Java Code Examples for org.apache.nifi.util.file.FileUtils#deleteFile()

@Test
public void testSimpleWriteWithToc() throws IOException {
    final File journalFile = new File("target/storage/" + UUID.randomUUID().toString() + "/testSimpleWrite");
    final File tocFile = TocUtil.getTocFile(journalFile);
    final TocWriter tocWriter = new StandardTocWriter(tocFile, false, false);
    final RecordWriter writer = createWriter(journalFile, tocWriter, false, 1024 * 1024);

    writer.writeHeader(1L);
    writer.writeRecord(createEvent());
    writer.close();

    final TocReader tocReader = new StandardTocReader(tocFile);

    try (final FileInputStream fis = new FileInputStream(journalFile);
        final RecordReader reader = createReader(fis, journalFile.getName(), tocReader, 2048)) {
        assertEquals(0, reader.getBlockIndex());
        reader.skipToBlock(0);
        final StandardProvenanceEventRecord recovered = reader.nextRecord();
        assertNotNull(recovered);

        assertEquals("nifi://unit-test", recovered.getTransitUri());
        assertNull(reader.nextRecord());
    }

    FileUtils.deleteFile(journalFile.getParentFile(), true);
}
 

@Test
public void testSingleRecordCompressed() throws IOException {
    final File journalFile = new File("target/storage/" + UUID.randomUUID().toString() + "/testSimpleWrite.gz");
    final File tocFile = TocUtil.getTocFile(journalFile);
    final TocWriter tocWriter = new StandardTocWriter(tocFile, false, false);
    final RecordWriter writer = createWriter(journalFile, tocWriter, true, 8192);

    writer.writeHeader(1L);
    writer.writeRecord(createEvent());
    writer.close();

    final TocReader tocReader = new StandardTocReader(tocFile);

    try (final FileInputStream fis = new FileInputStream(journalFile);
        final RecordReader reader = createReader(fis, journalFile.getName(), tocReader, 2048)) {
        assertEquals(0, reader.getBlockIndex());
        reader.skipToBlock(0);
        final StandardProvenanceEventRecord recovered = reader.nextRecord();
        assertNotNull(recovered);

        assertEquals("nifi://unit-test", recovered.getTransitUri());
        assertNull(reader.nextRecord());
    }

    FileUtils.deleteFile(journalFile.getParentFile(), true);
}
 

@Test
public void testOverwriteEmptyFile() throws IOException {
    final File tocFile = new File("target/" + UUID.randomUUID().toString() + ".toc");
    try {
        assertTrue( tocFile.createNewFile() );

        try (final StandardTocWriter writer = new StandardTocWriter(tocFile, false, false)) {
        }
    } finally {
        FileUtils.deleteFile(tocFile, false);
    }
}
 

public void replaceDirectory(final File oldIndexDir, final File newIndexDir, final boolean destroyOldIndex) {
    boolean replaced = false;

    synchronized (this) {
        for (final Map.Entry<Long, List<IndexLocation>> entry : indexLocationByTimestamp.entrySet()) {
            final List<IndexLocation> locations = entry.getValue();
            final ListIterator<IndexLocation> itr = locations.listIterator();

            while (itr.hasNext()) {
                final IndexLocation location = itr.next();
                if (location.getIndexDirectory().equals(oldIndexDir)) {
                    final IndexLocation updatedLocation = new IndexLocation(newIndexDir, location.getIndexStartTimestamp(), location.getPartitionName());
                    itr.set(updatedLocation);
                    replaced = true;
                    logger.debug("Replaced {} with {}", location, updatedLocation);
                }
            }
        }
    }

    if (!replaced) {
        insertIndexDirectory(newIndexDir);
    }

    if (destroyOldIndex) {
        try {
            FileUtils.deleteFile(oldIndexDir, true);
        } catch (IOException e) {
            logger.warn("Failed to delete index directory {}; this directory should be cleaned up manually", oldIndexDir, e);
        }
    }

    removeDirectory(oldIndexDir);

    logger.info("Successfully replaced old index directory {} with new index directory {}", oldIndexDir, newIndexDir);
}
 

@BeforeClass
public static void setupBeforeClass() throws IOException {
    System.setProperty("derby.stream.error.file", "target/derby.log");

    // remove previous test database, if any
    final File dbLocation = new File(DB_LOCATION);
    try {
        FileUtils.deleteFile(dbLocation, true);
    } catch (IOException ioe) {
        // Do nothing, may not have existed
    }
}
 

@AfterClass
public static void cleanUpAfterClass() throws Exception {
    try {
        DriverManager.getConnection("jdbc:derby:" + DB_LOCATION + ";shutdown=true");
    } catch (SQLNonTransientConnectionException e) {
        // Do nothing, this is what happens at Derby shutdown
    }
    // remove previous test database, if any
    final File dbLocation = new File(DB_LOCATION);
    try {
        FileUtils.deleteFile(dbLocation, true);
    } catch (IOException ioe) {
        // Do nothing, may not have existed
    }
}
 

protected boolean tryDeleteIndex(final File indexDirectory) {
    final long startNanos = System.nanoTime();
    boolean removed = false;
    while (!removed && System.nanoTime() - startNanos < TimeUnit.SECONDS.toNanos(MAX_DELETE_INDEX_WAIT_SECONDS)) {
        removed = indexManager.removeIndex(indexDirectory);

        if (!removed) {
            try {
                Thread.sleep(5000L);
            } catch (final InterruptedException ie) {
                logger.debug("Interrupted when trying to remove index {} from IndexManager; will not remove index", indexDirectory);
                Thread.currentThread().interrupt();
                return false;
            }
        }
    }

    if (removed) {
        try {
            FileUtils.deleteFile(indexDirectory, true);
            logger.debug("Successfully deleted directory {}", indexDirectory);
        } catch (final IOException e) {
            logger.warn("The Lucene Index located at " + indexDirectory + " has expired and contains no Provenance Events that still exist in the respository. "
                + "However, the directory could not be deleted.", e);
        }

        directoryManager.removeDirectory(indexDirectory);
        logger.info("Successfully removed expired Lucene Index {}", indexDirectory);
    } else {
        logger.warn("The Lucene Index located at {} has expired and contains no Provenance Events that still exist in the respository. "
            + "However, the directory could not be deleted because it is still actively being used. Will continue to try to delete "
            + "in a subsequent maintenance cycle.", indexDirectory);
    }

    return removed;
}
 

@BeforeClass
public static void setupBeforeClass() throws IOException {
    System.setProperty("derby.stream.error.file", "target/derby.log");

    // remove previous test database, if any
    final File dbLocation = new File(DB_LOCATION);
    try {
        FileUtils.deleteFile(dbLocation, true);
    } catch (IOException ioe) {
        // Do nothing, may not have existed
    }
}
 

@Test
public void testMultipleRecordsMultipleBlocksCompressed() throws IOException {
    final File journalFile = new File("target/storage/" + UUID.randomUUID().toString() + "/testSimpleWrite.gz");
    final File tocFile = TocUtil.getTocFile(journalFile);
    final TocWriter tocWriter = new StandardTocWriter(tocFile, false, false);
    // new block each 10 bytes
    final RecordWriter writer = createWriter(journalFile, tocWriter, true, 100);

    writer.writeHeader(1L);
    for (int i = 0; i < 10; i++) {
        writer.writeRecord(createEvent());
    }
    writer.close();

    final TocReader tocReader = new StandardTocReader(tocFile);

    try (final FileInputStream fis = new FileInputStream(journalFile);
        final RecordReader reader = createReader(fis, journalFile.getName(), tocReader, 2048)) {
        for (int i = 0; i < 10; i++) {
            final StandardProvenanceEventRecord recovered = reader.nextRecord();
            System.out.println(recovered);
            assertNotNull(recovered);
            assertEquals(i, recovered.getEventId());
            assertEquals("nifi://unit-test", recovered.getTransitUri());

            final Map<String, String> updatedAttrs = recovered.getUpdatedAttributes();
            assertNotNull(updatedAttrs);
            assertEquals(2, updatedAttrs.size());
            assertEquals("1.txt", updatedAttrs.get("filename"));
            assertTrue(updatedAttrs.containsKey("uuid"));
        }

        assertNull(reader.nextRecord());
    }

    FileUtils.deleteFile(journalFile.getParentFile(), true);
}
 

@Test
public void testMultipleWritersSimultaneouslySameIndex() throws IOException {
    final SimpleIndexManager mgr = new SimpleIndexManager(new RepositoryConfiguration());
    final File dir = new File("target/" + UUID.randomUUID().toString());
    try {
        final EventIndexWriter writer1 = mgr.borrowIndexWriter(dir);
        final EventIndexWriter writer2 = mgr.borrowIndexWriter(dir);

        final Document doc1 = new Document();
        doc1.add(new StringField("id", "1", Store.YES));

        final Document doc2 = new Document();
        doc2.add(new StringField("id", "2", Store.YES));

        writer1.index(doc1, 1000);
        writer2.index(doc2, 1000);
        mgr.returnIndexWriter(writer2);
        mgr.returnIndexWriter(writer1);

        final EventIndexSearcher searcher = mgr.borrowIndexSearcher(dir);
        final TopDocs topDocs = searcher.getIndexSearcher().search(new MatchAllDocsQuery(), 2);
        assertEquals(2, topDocs.totalHits);
        mgr.returnIndexSearcher(searcher);
    } finally {
        FileUtils.deleteFile(dir, true);
    }
}
 

@Test
public void testMultipleRecordsMultipleBlocksCompressed() throws IOException {
    final File journalFile = new File("target/storage/" + UUID.randomUUID().toString() + "/testSimpleWrite.gz");
    final File tocFile = TocUtil.getTocFile(journalFile);
    final TocWriter tocWriter = new StandardTocWriter(tocFile, false, false);
    // new block each 10 bytes
    final RecordWriter writer = createWriter(journalFile, tocWriter, true, 100);

    writer.writeHeader(1L);
    for (int i = 0; i < 10; i++) {
        writer.writeRecord(createEvent());
    }
    writer.close();

    final TocReader tocReader = new StandardTocReader(tocFile);

    try (final FileInputStream fis = new FileInputStream(journalFile);
        final RecordReader reader = createReader(fis, journalFile.getName(), tocReader, 2048)) {
        for (int i = 0; i < 10; i++) {
            final StandardProvenanceEventRecord recovered = reader.nextRecord();
            System.out.println(recovered);
            assertNotNull(recovered);
            assertEquals(i, recovered.getEventId());
            assertEquals("nifi://unit-test", recovered.getTransitUri());

            final Map<String, String> updatedAttrs = recovered.getUpdatedAttributes();
            assertNotNull(updatedAttrs);
            assertEquals(2, updatedAttrs.size());
            assertEquals("1.txt", updatedAttrs.get("filename"));
            assertTrue(updatedAttrs.containsKey("uuid"));
        }

        assertNull(reader.nextRecord());
    }

    FileUtils.deleteFile(journalFile.getParentFile(), true);
}
 

public void clearRepo() throws IOException {
    final File target = new File("target");
    final File testRepo = new File(target, "test-repo");
    if (testRepo.exists()) {
        FileUtils.deleteFile(testRepo, true);
    }
}
 

@AfterClass
public static void cleanUpAfterClass() throws Exception {
    try {
        DriverManager.getConnection("jdbc:derby:" + DB_LOCATION + ";shutdown=true");
    } catch (SQLNonTransientConnectionException e) {
        // Do nothing, this is what happens at Derby shutdown
    }
    // remove previous test database, if any
    final File dbLocation = new File(DB_LOCATION);
    try {
        FileUtils.deleteFile(dbLocation, true);
    } catch (IOException ioe) {
        // Do nothing, may not have existed
    }
}
 

private static boolean deleteFile(final File file) {
    if (file.isDirectory()) {
        FileUtils.deleteFilesInDir(file, null, null, true, true);
    }
    return FileUtils.deleteFile(file, null, 10);
}
 

private static boolean deleteFile(final File file) {
    if (file.isDirectory()) {
        FileUtils.deleteFilesInDir(file, null, null, true, true);
    }
    return FileUtils.deleteFile(file, null, 10);
}
 

@Test
public void testContentClaimRemoved() throws IOException {
    final File journalFile = new File("target/storage/" + UUID.randomUUID().toString() + "/testSimpleWrite.gz");
    final File tocFile = TocUtil.getTocFile(journalFile);
    final TocWriter tocWriter = new StandardTocWriter(tocFile, false, false);
    final RecordWriter writer = createWriter(journalFile, tocWriter, true, 8192);

    final Map<String, String> attributes = new HashMap<>();
    attributes.put("filename", "1.txt");
    attributes.put("uuid", UUID.randomUUID().toString());

    final ProvenanceEventBuilder builder = new StandardProvenanceEventRecord.Builder();
    builder.setEventTime(System.currentTimeMillis());
    builder.setEventType(ProvenanceEventType.RECEIVE);
    builder.setTransitUri("nifi://unit-test");
    builder.fromFlowFile(TestUtil.createFlowFile(3L, 3000L, attributes));
    builder.setComponentId("1234");
    builder.setComponentType("dummy processor");
    builder.setPreviousContentClaim("container-1", "section-1", "identifier-1", 1L, 1L);
    builder.setCurrentContentClaim(null, null, null, 0L, 0L);
    final ProvenanceEventRecord record = builder.build();

    writer.writeHeader(1L);
    writer.writeRecord(record);
    writer.close();

    final TocReader tocReader = new StandardTocReader(tocFile);

    try (final FileInputStream fis = new FileInputStream(journalFile);
        final RecordReader reader = createReader(fis, journalFile.getName(), tocReader, 2048)) {
        assertEquals(0, reader.getBlockIndex());
        reader.skipToBlock(0);
        final StandardProvenanceEventRecord recovered = reader.nextRecord();
        assertNotNull(recovered);

        assertEquals("nifi://unit-test", recovered.getTransitUri());

        assertEquals("container-1", recovered.getPreviousContentClaimContainer());
        assertNull(recovered.getContentClaimContainer());

        assertEquals("section-1", recovered.getPreviousContentClaimSection());
        assertNull(recovered.getContentClaimSection());

        assertEquals("identifier-1", recovered.getPreviousContentClaimIdentifier());
        assertNull(recovered.getContentClaimIdentifier());

        assertEquals(1L, recovered.getPreviousContentClaimOffset().longValue());
        assertNull(recovered.getContentClaimOffset());

        assertEquals(1L, recovered.getPreviousFileSize().longValue());
        assertEquals(0L, recovered.getFileSize());

        assertNull(reader.nextRecord());
    }

    FileUtils.deleteFile(journalFile.getParentFile(), true);
}
 

private static boolean deleteFile(final File file) {
    if (file.isDirectory()) {
        FileUtils.deleteFilesInDir(file, null, null, true, true);
    }
    return FileUtils.deleteFile(file, null, 10);
}
 

@Test
public void testContentClaimChanged() throws IOException {
    final File journalFile = new File("target/storage/" + UUID.randomUUID().toString() + "/testSimpleWrite.gz");
    final File tocFile = TocUtil.getTocFile(journalFile);
    final TocWriter tocWriter = new StandardTocWriter(tocFile, false, false);
    final RecordWriter writer = createWriter(journalFile, tocWriter, true, 8192);

    final Map<String, String> attributes = new HashMap<>();
    attributes.put("filename", "1.txt");
    attributes.put("uuid", UUID.randomUUID().toString());

    final ProvenanceEventBuilder builder = new StandardProvenanceEventRecord.Builder();
    builder.setEventTime(System.currentTimeMillis());
    builder.setEventType(ProvenanceEventType.RECEIVE);
    builder.setTransitUri("nifi://unit-test");
    builder.fromFlowFile(TestUtil.createFlowFile(3L, 3000L, attributes));
    builder.setComponentId("1234");
    builder.setComponentType("dummy processor");
    builder.setPreviousContentClaim("container-1", "section-1", "identifier-1", 1L, 1L);
    builder.setCurrentContentClaim("container-2", "section-2", "identifier-2", 2L, 2L);
    final ProvenanceEventRecord record = builder.build();

    writer.writeHeader(1L);
    writer.writeRecord(record);
    writer.close();

    final TocReader tocReader = new StandardTocReader(tocFile);

    try (final FileInputStream fis = new FileInputStream(journalFile);
        final RecordReader reader = createReader(fis, journalFile.getName(), tocReader, 2048)) {
        assertEquals(0, reader.getBlockIndex());
        reader.skipToBlock(0);
        final StandardProvenanceEventRecord recovered = reader.nextRecord();
        assertNotNull(recovered);

        assertEquals("nifi://unit-test", recovered.getTransitUri());

        assertEquals("container-1", recovered.getPreviousContentClaimContainer());
        assertEquals("container-2", recovered.getContentClaimContainer());

        assertEquals("section-1", recovered.getPreviousContentClaimSection());
        assertEquals("section-2", recovered.getContentClaimSection());

        assertEquals("identifier-1", recovered.getPreviousContentClaimIdentifier());
        assertEquals("identifier-2", recovered.getContentClaimIdentifier());

        assertEquals(1L, recovered.getPreviousContentClaimOffset().longValue());
        assertEquals(2L, recovered.getContentClaimOffset().longValue());

        assertEquals(1L, recovered.getPreviousFileSize().longValue());
        assertEquals(2L, recovered.getContentClaimOffset().longValue());

        assertNull(reader.nextRecord());
    }

    FileUtils.deleteFile(journalFile.getParentFile(), true);
}
 

@Test
public void testRestartWithOneRecord() throws IOException {
    final Path path = Paths.get("target/test-repo");
    if (Files.exists(path)) {
        FileUtils.deleteFile(path.toFile(), true);
    }

    final WriteAheadFlowFileRepository repo = new WriteAheadFlowFileRepository(niFiProperties);
    repo.initialize(new StandardResourceClaimManager());

    final TestQueueProvider queueProvider = new TestQueueProvider();
    repo.loadFlowFiles(queueProvider);

    final List<FlowFileRecord> flowFileCollection = new ArrayList<>();

    final Connection connection = Mockito.mock(Connection.class);
    when(connection.getIdentifier()).thenReturn("1234");

    final FlowFileQueue queue = Mockito.mock(FlowFileQueue.class);
    when(queue.getIdentifier()).thenReturn("1234");
    doAnswer(new Answer<Object>() {
        @Override
        public Object answer(final InvocationOnMock invocation) throws Throwable {
            flowFileCollection.add((FlowFileRecord) invocation.getArguments()[0]);
            return null;
        }
    }).when(queue).put(any(FlowFileRecord.class));

    when(connection.getFlowFileQueue()).thenReturn(queue);

    queueProvider.addConnection(connection);

    StandardFlowFileRecord.Builder ffBuilder = new StandardFlowFileRecord.Builder();
    ffBuilder.id(1L);
    ffBuilder.addAttribute("abc", "xyz");
    ffBuilder.size(0L);
    final FlowFileRecord flowFileRecord = ffBuilder.build();

    final List<RepositoryRecord> records = new ArrayList<>();
    final StandardRepositoryRecord record = new StandardRepositoryRecord(null);
    record.setWorking(flowFileRecord);
    record.setDestination(connection.getFlowFileQueue());
    records.add(record);

    repo.updateRepository(records);

    // update to add new attribute
    ffBuilder = new StandardFlowFileRecord.Builder().fromFlowFile(flowFileRecord).addAttribute("hello", "world");
    final FlowFileRecord flowFileRecord2 = ffBuilder.build();
    record.setWorking(flowFileRecord2);
    repo.updateRepository(records);

    // update size but no attribute
    ffBuilder = new StandardFlowFileRecord.Builder().fromFlowFile(flowFileRecord2).size(40L);
    final FlowFileRecord flowFileRecord3 = ffBuilder.build();
    record.setWorking(flowFileRecord3);
    repo.updateRepository(records);

    repo.close();

    // restore
    final WriteAheadFlowFileRepository repo2 = new WriteAheadFlowFileRepository(niFiProperties);
    repo2.initialize(new StandardResourceClaimManager());
    repo2.loadFlowFiles(queueProvider);

    assertEquals(1, flowFileCollection.size());
    final FlowFileRecord flowFile = flowFileCollection.get(0);
    assertEquals(1L, flowFile.getId());
    assertEquals("xyz", flowFile.getAttribute("abc"));
    assertEquals(40L, flowFile.getSize());
    assertEquals("world", flowFile.getAttribute("hello"));

    repo2.close();
}
 

@Test
public void testContentClaimUnchanged() throws IOException {
    final File journalFile = new File("target/storage/" + UUID.randomUUID().toString() + "/testSimpleWrite.gz");
    final File tocFile = TocUtil.getTocFile(journalFile);
    final TocWriter tocWriter = new StandardTocWriter(tocFile, false, false);
    final RecordWriter writer = createWriter(journalFile, tocWriter, true, 8192);

    final Map<String, String> attributes = new HashMap<>();
    attributes.put("filename", "1.txt");
    attributes.put("uuid", UUID.randomUUID().toString());

    final ProvenanceEventBuilder builder = new StandardProvenanceEventRecord.Builder();
    builder.setEventTime(System.currentTimeMillis());
    builder.setEventType(ProvenanceEventType.RECEIVE);
    builder.setTransitUri("nifi://unit-test");
    builder.fromFlowFile(TestUtil.createFlowFile(3L, 3000L, attributes));
    builder.setComponentId("1234");
    builder.setComponentType("dummy processor");
    builder.setPreviousContentClaim("container-1", "section-1", "identifier-1", 1L, 1L);
    builder.setCurrentContentClaim("container-1", "section-1", "identifier-1", 1L, 1L);
    final ProvenanceEventRecord record = builder.build();

    writer.writeHeader(1L);
    writer.writeRecord(record);
    writer.close();

    final TocReader tocReader = new StandardTocReader(tocFile);

    try (final FileInputStream fis = new FileInputStream(journalFile);
        final RecordReader reader = createReader(fis, journalFile.getName(), tocReader, 2048)) {
        assertEquals(0, reader.getBlockIndex());
        reader.skipToBlock(0);
        final StandardProvenanceEventRecord recovered = reader.nextRecord();
        assertNotNull(recovered);

        assertEquals("nifi://unit-test", recovered.getTransitUri());

        assertEquals("container-1", recovered.getPreviousContentClaimContainer());
        assertEquals("container-1", recovered.getContentClaimContainer());

        assertEquals("section-1", recovered.getPreviousContentClaimSection());
        assertEquals("section-1", recovered.getContentClaimSection());

        assertEquals("identifier-1", recovered.getPreviousContentClaimIdentifier());
        assertEquals("identifier-1", recovered.getContentClaimIdentifier());

        assertEquals(1L, recovered.getPreviousContentClaimOffset().longValue());
        assertEquals(1L, recovered.getContentClaimOffset().longValue());

        assertEquals(1L, recovered.getPreviousFileSize().longValue());
        assertEquals(1L, recovered.getContentClaimOffset().longValue());

        assertNull(reader.nextRecord());
    }

    FileUtils.deleteFile(journalFile.getParentFile(), true);
}