Java Code Examples for org.apache.nifi.provenance.toc.TocUtil

The following examples show how to use org.apache.nifi.provenance.toc.TocUtil. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: localization_nifi   Source File: FileRemovalAction.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public File execute(final File expiredFile) throws IOException {
    final boolean removed = remove(expiredFile);
    if (removed) {
        logger.info("Removed expired Provenance Event file {}", expiredFile);
    } else {
        logger.warn("Failed to remove old Provenance Event file {}; this file should be cleaned up manually", expiredFile);
    }

    final File tocFile = TocUtil.getTocFile(expiredFile);
    if (remove(tocFile)) {
        logger.info("Removed expired Provenance Table-of-Contents file {}", tocFile);
    } else {
        logger.warn("Failed to remove old Provenance Table-of-Contents file {}; this file should be cleaned up manually", expiredFile);
    }

    return removed ? null : expiredFile;
}
 
Example 2
private boolean delete(final File file) {
    final long firstEventId = DirectoryUtils.getMinId(file);
    synchronized (minEventIdToPathMap) {
        minEventIdToPathMap.remove(firstEventId);
    }

    if (!file.delete()) {
        logger.warn("Failed to remove Provenance Event file {}; this file should be cleaned up manually", file);
        return false;
    }

    final File tocFile = TocUtil.getTocFile(file);
    if (tocFile.exists() && !tocFile.delete()) {
        logger.warn("Failed to remove Provenance Table-of-Contents file {}; this file should be cleaned up manually", tocFile);
    }

    return true;
}
 
Example 3
@Test
public void testSimpleWriteWithToc() throws IOException {
    final File journalFile = new File("target/storage/" + UUID.randomUUID().toString() + "/testSimpleWrite");
    final File tocFile = TocUtil.getTocFile(journalFile);
    final TocWriter tocWriter = new StandardTocWriter(tocFile, false, false);
    final RecordWriter writer = createWriter(journalFile, tocWriter, false, 1024 * 1024);

    writer.writeHeader(1L);
    writer.writeRecord(createEvent());
    writer.close();

    final TocReader tocReader = new StandardTocReader(tocFile);

    try (final FileInputStream fis = new FileInputStream(journalFile);
        final RecordReader reader = createReader(fis, journalFile.getName(), tocReader, 2048)) {
        assertEquals(0, reader.getBlockIndex());
        reader.skipToBlock(0);
        final StandardProvenanceEventRecord recovered = reader.nextRecord();
        assertNotNull(recovered);

        assertEquals("nifi://unit-test", recovered.getTransitUri());
        assertNull(reader.nextRecord());
    }

    FileUtils.deleteFile(journalFile.getParentFile(), true);
}
 
Example 4
@Test
public void testSingleRecordCompressed() throws IOException {
    final File journalFile = new File("target/storage/" + UUID.randomUUID().toString() + "/testSimpleWrite.gz");
    final File tocFile = TocUtil.getTocFile(journalFile);
    final TocWriter tocWriter = new StandardTocWriter(tocFile, false, false);
    final RecordWriter writer = createWriter(journalFile, tocWriter, true, 8192);

    writer.writeHeader(1L);
    writer.writeRecord(createEvent());
    writer.close();

    final TocReader tocReader = new StandardTocReader(tocFile);

    try (final FileInputStream fis = new FileInputStream(journalFile);
        final RecordReader reader = createReader(fis, journalFile.getName(), tocReader, 2048)) {
        assertEquals(0, reader.getBlockIndex());
        reader.skipToBlock(0);
        final StandardProvenanceEventRecord recovered = reader.nextRecord();
        assertNotNull(recovered);

        assertEquals("nifi://unit-test", recovered.getTransitUri());
        assertNull(reader.nextRecord());
    }

    FileUtils.deleteFile(journalFile.getParentFile(), true);
}
 
Example 5
Source Project: nifi   Source File: FileRemovalAction.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public File execute(final File expiredFile) throws IOException {
    final boolean removed = remove(expiredFile);
    if (removed) {
        logger.info("Removed expired Provenance Event file {}", expiredFile);
    } else {
        logger.warn("Failed to remove old Provenance Event file {}; this file should be cleaned up manually", expiredFile);
    }

    final File tocFile = TocUtil.getTocFile(expiredFile);
    if (remove(tocFile)) {
        logger.info("Removed expired Provenance Table-of-Contents file {}", tocFile);
    } else {
        logger.warn("Failed to remove old Provenance Table-of-Contents file {}; this file should be cleaned up manually", expiredFile);
    }

    return removed ? null : expiredFile;
}
 
Example 6
Source Project: nifi   Source File: WriteAheadProvenanceRepository.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public synchronized void initialize(final EventReporter eventReporter, final Authorizer authorizer, final ProvenanceAuthorizableFactory resourceFactory,
    final IdentifierLookup idLookup) throws IOException {
    final RecordWriterFactory recordWriterFactory = (file, idGenerator, compressed, createToc) -> {
        final TocWriter tocWriter = createToc ? new StandardTocWriter(TocUtil.getTocFile(file), false, false) : null;
        return new EventIdFirstSchemaRecordWriter(file, idGenerator, tocWriter, compressed, BLOCK_SIZE, idLookup);
    };

    final EventFileManager fileManager = new EventFileManager();
    final RecordReaderFactory recordReaderFactory = (file, logs, maxChars) -> {
        fileManager.obtainReadLock(file);
        try {
            return RecordReaders.newRecordReader(file, logs, maxChars);
        } finally {
            fileManager.releaseReadLock(file);
        }
    };

   init(recordWriterFactory, recordReaderFactory, eventReporter, authorizer, resourceFactory);
}
 
Example 7
Source Project: nifi   Source File: AbstractTestRecordReaderWriter.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testSimpleWriteWithToc() throws IOException {
    final File journalFile = new File("target/storage/" + UUID.randomUUID().toString() + "/testSimpleWrite");
    final File tocFile = TocUtil.getTocFile(journalFile);
    final TocWriter tocWriter = new StandardTocWriter(tocFile, false, false);
    final RecordWriter writer = createWriter(journalFile, tocWriter, false, 1024 * 1024);

    writer.writeHeader(1L);
    writer.writeRecord(createEvent());
    writer.close();

    final TocReader tocReader = new StandardTocReader(tocFile);
    final String expectedTransitUri = "nifi://unit-test";
        final int expectedBlockIndex = 0;

    assertRecoveredRecord(journalFile, tocReader, expectedTransitUri, expectedBlockIndex);

    FileUtils.deleteFile(journalFile.getParentFile(), true);
}
 
Example 8
Source Project: nifi   Source File: AbstractTestRecordReaderWriter.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testSingleRecordCompressed() throws IOException {
    final File journalFile = new File("target/storage/" + UUID.randomUUID().toString() + "/testSimpleWrite.gz");
    final File tocFile = TocUtil.getTocFile(journalFile);
    final TocWriter tocWriter = new StandardTocWriter(tocFile, false, false);
    final RecordWriter writer = createWriter(journalFile, tocWriter, true, 8192);

    writer.writeHeader(1L);
    writer.writeRecord(createEvent());
    writer.close();

    final TocReader tocReader = new StandardTocReader(tocFile);

    assertRecoveredRecord(journalFile, tocReader, "nifi://unit-test", 0);

    FileUtils.deleteFile(journalFile.getParentFile(), true);
}
 
Example 9
@Override
public synchronized void initialize(final EventReporter eventReporter, final Authorizer authorizer, final ProvenanceAuthorizableFactory resourceFactory,
    final IdentifierLookup idLookup) throws IOException {
    final RecordWriterFactory recordWriterFactory = (file, idGenerator, compressed, createToc) -> {
        final TocWriter tocWriter = createToc ? new StandardTocWriter(TocUtil.getTocFile(file), false, false) : null;
        return new EventIdFirstSchemaRecordWriter(file, idGenerator, tocWriter, compressed, BLOCK_SIZE, idLookup);
    };

    final EventFileManager fileManager = new EventFileManager();
    final RecordReaderFactory recordReaderFactory = (file, logs, maxChars) -> {
        fileManager.obtainReadLock(file);
        try {
            return RecordReaders.newRecordReader(file, logs, maxChars);
        } finally {
            fileManager.releaseReadLock(file);
        }
    };

    eventStore = new PartitionedWriteAheadEventStore(config, recordWriterFactory, recordReaderFactory, eventReporter, fileManager);

    final IndexManager indexManager = new SimpleIndexManager(config);
    eventIndex = new LuceneEventIndex(config, indexManager, eventReporter);

    this.eventReporter = eventReporter;
    this.authorizer = authorizer;
    this.resourceFactory = resourceFactory;

    eventStore.initialize();
    eventIndex.initialize(eventStore);

    try {
        eventStore.reindexLatestEvents(eventIndex);
    } catch (final Exception e) {
        logger.error("Failed to re-index some of the Provenance Events. It is possible that some of the latest "
            + "events will not be available from the Provenance Repository when a query is issued.", e);
    }
}
 
Example 10
@Test
public void testMultipleRecordsSameBlockCompressed() throws IOException {
    final File journalFile = new File("target/storage/" + UUID.randomUUID().toString() + "/testSimpleWrite.gz");
    final File tocFile = TocUtil.getTocFile(journalFile);
    final TocWriter tocWriter = new StandardTocWriter(tocFile, false, false);
    // new record each 1 MB of uncompressed data
    final RecordWriter writer = createWriter(journalFile, tocWriter, true, 1024 * 1024);

    writer.writeHeader(1L);
    for (int i = 0; i < 10; i++) {
        writer.writeRecord(createEvent());
    }
    writer.close();

    final TocReader tocReader = new StandardTocReader(tocFile);

    try (final FileInputStream fis = new FileInputStream(journalFile);
        final RecordReader reader = createReader(fis, journalFile.getName(), tocReader, 2048)) {
        for (int i = 0; i < 10; i++) {
            assertEquals(0, reader.getBlockIndex());

            // call skipToBlock half the time to ensure that we can; avoid calling it
            // the other half of the time to ensure that it's okay.
            if (i <= 5) {
                reader.skipToBlock(0);
            }

            final StandardProvenanceEventRecord recovered = reader.nextRecord();
            assertNotNull(recovered);
            assertEquals("nifi://unit-test", recovered.getTransitUri());
        }

        assertNull(reader.nextRecord());
    }

    FileUtils.deleteFile(journalFile.getParentFile(), true);
}
 
Example 11
@Test
public void testMultipleRecordsMultipleBlocksCompressed() throws IOException {
    final File journalFile = new File("target/storage/" + UUID.randomUUID().toString() + "/testSimpleWrite.gz");
    final File tocFile = TocUtil.getTocFile(journalFile);
    final TocWriter tocWriter = new StandardTocWriter(tocFile, false, false);
    // new block each 10 bytes
    final RecordWriter writer = createWriter(journalFile, tocWriter, true, 100);

    writer.writeHeader(1L);
    for (int i = 0; i < 10; i++) {
        writer.writeRecord(createEvent());
    }
    writer.close();

    final TocReader tocReader = new StandardTocReader(tocFile);

    try (final FileInputStream fis = new FileInputStream(journalFile);
        final RecordReader reader = createReader(fis, journalFile.getName(), tocReader, 2048)) {
        for (int i = 0; i < 10; i++) {
            final StandardProvenanceEventRecord recovered = reader.nextRecord();
            System.out.println(recovered);
            assertNotNull(recovered);
            assertEquals(i, recovered.getEventId());
            assertEquals("nifi://unit-test", recovered.getTransitUri());

            final Map<String, String> updatedAttrs = recovered.getUpdatedAttributes();
            assertNotNull(updatedAttrs);
            assertEquals(2, updatedAttrs.size());
            assertEquals("1.txt", updatedAttrs.get("filename"));
            assertTrue(updatedAttrs.containsKey("uuid"));
        }

        assertNull(reader.nextRecord());
    }

    FileUtils.deleteFile(journalFile.getParentFile(), true);
}
 
Example 12
Source Project: nifi   Source File: WriteAheadStorePartition.java    License: Apache License 2.0 5 votes vote down vote up
private boolean delete(final File file) {
    final File activeEventFile = getActiveEventFile();
    if (file.equals(activeEventFile)) {
        logger.debug("Attempting to age off Active Event File {}. Will return without deleting the file.", file);
        return false;
    }

    final long firstEventId = DirectoryUtils.getMinId(file);
    synchronized (minEventIdToPathMap) {
        minEventIdToPathMap.remove(firstEventId);
    }

    eventFileManager.obtainWriteLock(file);
    try {
        if (!file.delete()) {
            logger.warn("Failed to remove Provenance Event file {}; this file should be cleaned up manually", file);
            return false;
        }

        final File tocFile = TocUtil.getTocFile(file);
        if (tocFile.exists() && !tocFile.delete()) {
            logger.warn("Failed to remove Provenance Table-of-Contents file {}; this file should be cleaned up manually", tocFile);
        }

        return true;
    } finally {
        eventFileManager.releaseWriteLock(file);
    }
}
 
Example 13
Source Project: nifi   Source File: AbstractTestRecordReaderWriter.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testMultipleRecordsSameBlockCompressed() throws IOException {
    final File journalFile = new File("target/storage/" + UUID.randomUUID().toString() + "/testSimpleWrite.gz");
    final File tocFile = TocUtil.getTocFile(journalFile);
    final TocWriter tocWriter = new StandardTocWriter(tocFile, false, false);
    // new record each 1 MB of uncompressed data
    final RecordWriter writer = createWriter(journalFile, tocWriter, true, 1024 * 1024);

    writer.writeHeader(1L);
    for (int i = 0; i < 10; i++) {
        writer.writeRecord(createEvent());
    }
    writer.close();

    final TocReader tocReader = new StandardTocReader(tocFile);

    try (final FileInputStream fis = new FileInputStream(journalFile);
        final RecordReader reader = createReader(fis, journalFile.getName(), tocReader, 2048)) {
        for (int i = 0; i < 10; i++) {
            assertEquals(0, reader.getBlockIndex());

            // call skipToBlock half the time to ensure that we can; avoid calling it
            // the other half of the time to ensure that it's okay.
            if (i <= 5) {
                reader.skipToBlock(0);
            }

            final StandardProvenanceEventRecord recovered = reader.nextRecord();
            assertNotNull(recovered);
            assertEquals("nifi://unit-test", recovered.getTransitUri());
        }

        assertNull(reader.nextRecord());
    }

    FileUtils.deleteFile(journalFile.getParentFile(), true);
}
 
Example 14
Source Project: nifi   Source File: AbstractTestRecordReaderWriter.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testMultipleRecordsMultipleBlocksCompressed() throws IOException {
    final File journalFile = new File("target/storage/" + UUID.randomUUID().toString() + "/testSimpleWrite.gz");
    final File tocFile = TocUtil.getTocFile(journalFile);
    final TocWriter tocWriter = new StandardTocWriter(tocFile, false, false);
    // new block each 10 bytes
    final RecordWriter writer = createWriter(journalFile, tocWriter, true, 100);

    writer.writeHeader(1L);
    for (int i = 0; i < 10; i++) {
        writer.writeRecord(createEvent());
    }
    writer.close();

    final TocReader tocReader = new StandardTocReader(tocFile);

    try (final FileInputStream fis = new FileInputStream(journalFile);
        final RecordReader reader = createReader(fis, journalFile.getName(), tocReader, 2048)) {
        for (int i = 0; i < 10; i++) {
            final StandardProvenanceEventRecord recovered = reader.nextRecord();
            System.out.println(recovered);
            assertNotNull(recovered);
            assertEquals(i, recovered.getEventId());
            assertEquals("nifi://unit-test", recovered.getTransitUri());

            final Map<String, String> updatedAttrs = recovered.getUpdatedAttributes();
            assertNotNull(updatedAttrs);
            assertEquals(2, updatedAttrs.size());
            assertEquals("1.txt", updatedAttrs.get("filename"));
            assertTrue(updatedAttrs.containsKey("uuid"));
        }

        assertNull(reader.nextRecord());
    }

    FileUtils.deleteFile(journalFile.getParentFile(), true);
}
 
Example 15
Source Project: nifi   Source File: TestWriteAheadStorePartition.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testInitEmptyFile() throws IOException {
    final RepositoryConfiguration repoConfig = createConfig(1, "testInitEmptyFile");
    repoConfig.setMaxEventFileCount(5);

    final String partitionName = repoConfig.getStorageDirectories().keySet().iterator().next();
    final File storageDirectory = repoConfig.getStorageDirectories().values().iterator().next();

    final RecordWriterFactory recordWriterFactory = (file, idGenerator, compressed, createToc) -> {
        final TocWriter tocWriter = createToc ? new StandardTocWriter(TocUtil.getTocFile(file), false, false) : null;
        return new EventIdFirstSchemaRecordWriter(file, idGenerator, tocWriter, compressed, 32 * 1024, IdentifierLookup.EMPTY);
    };

    final RecordReaderFactory recordReaderFactory = RecordReaders::newRecordReader;

    WriteAheadStorePartition partition = new WriteAheadStorePartition(storageDirectory, partitionName, repoConfig, recordWriterFactory,
            recordReaderFactory, new LinkedBlockingQueue<>(), new AtomicLong(0L), EventReporter.NO_OP, Mockito.mock(EventFileManager.class));

    for (int i = 0; i < 100; i++) {
        partition.addEvents(Collections.singleton(TestUtil.createEvent()));
    }

    long maxEventId = partition.getMaxEventId();
    assertTrue(maxEventId > 0);
    partition.close();

    final List<File> fileList = Arrays.asList(storageDirectory.listFiles(DirectoryUtils.EVENT_FILE_FILTER));
    Collections.sort(fileList, DirectoryUtils.LARGEST_ID_FIRST);

    // Create new empty prov file with largest id
    assertTrue(new File(storageDirectory, "1" + fileList.get(0).getName()).createNewFile());

    partition = new WriteAheadStorePartition(storageDirectory, partitionName, repoConfig, recordWriterFactory,
            recordReaderFactory, new LinkedBlockingQueue<>(), new AtomicLong(0L), EventReporter.NO_OP, Mockito.mock(EventFileManager.class));

    partition.initialize();

    assertEquals(maxEventId, partition.getMaxEventId());
}
 
Example 16
Source Project: localization_nifi   Source File: RecordWriters.java    License: Apache License 2.0 4 votes vote down vote up
public static RecordWriter newSchemaRecordWriter(final File file, final AtomicLong idGenerator, final boolean compressed, final boolean createToc,
    final int compressionBlockBytes) throws IOException {
    final TocWriter tocWriter = createToc ? new StandardTocWriter(TocUtil.getTocFile(file), false, false) : null;
    return new ByteArraySchemaRecordWriter(file, idGenerator, tocWriter, compressed, compressionBlockBytes);
}
 
Example 17
@Before
public void setup() {
    journalFile = new File("target/storage/" + UUID.randomUUID().toString() + "/testEventIdFirstSchemaRecordReaderWriter");
    tocFile = TocUtil.getTocFile(journalFile);
    idGenerator.set(0L);
}
 
Example 18
@Test
public void testContentClaimUnchanged() throws IOException {
    final File journalFile = new File("target/storage/" + UUID.randomUUID().toString() + "/testSimpleWrite.gz");
    final File tocFile = TocUtil.getTocFile(journalFile);
    final TocWriter tocWriter = new StandardTocWriter(tocFile, false, false);
    final RecordWriter writer = createWriter(journalFile, tocWriter, true, 8192);

    final Map<String, String> attributes = new HashMap<>();
    attributes.put("filename", "1.txt");
    attributes.put("uuid", UUID.randomUUID().toString());

    final ProvenanceEventBuilder builder = new StandardProvenanceEventRecord.Builder();
    builder.setEventTime(System.currentTimeMillis());
    builder.setEventType(ProvenanceEventType.RECEIVE);
    builder.setTransitUri("nifi://unit-test");
    builder.fromFlowFile(TestUtil.createFlowFile(3L, 3000L, attributes));
    builder.setComponentId("1234");
    builder.setComponentType("dummy processor");
    builder.setPreviousContentClaim("container-1", "section-1", "identifier-1", 1L, 1L);
    builder.setCurrentContentClaim("container-1", "section-1", "identifier-1", 1L, 1L);
    final ProvenanceEventRecord record = builder.build();

    writer.writeHeader(1L);
    writer.writeRecord(record);
    writer.close();

    final TocReader tocReader = new StandardTocReader(tocFile);

    try (final FileInputStream fis = new FileInputStream(journalFile);
        final RecordReader reader = createReader(fis, journalFile.getName(), tocReader, 2048)) {
        assertEquals(0, reader.getBlockIndex());
        reader.skipToBlock(0);
        final StandardProvenanceEventRecord recovered = reader.nextRecord();
        assertNotNull(recovered);

        assertEquals("nifi://unit-test", recovered.getTransitUri());

        assertEquals("container-1", recovered.getPreviousContentClaimContainer());
        assertEquals("container-1", recovered.getContentClaimContainer());

        assertEquals("section-1", recovered.getPreviousContentClaimSection());
        assertEquals("section-1", recovered.getContentClaimSection());

        assertEquals("identifier-1", recovered.getPreviousContentClaimIdentifier());
        assertEquals("identifier-1", recovered.getContentClaimIdentifier());

        assertEquals(1L, recovered.getPreviousContentClaimOffset().longValue());
        assertEquals(1L, recovered.getContentClaimOffset().longValue());

        assertEquals(1L, recovered.getPreviousFileSize().longValue());
        assertEquals(1L, recovered.getContentClaimOffset().longValue());

        assertNull(reader.nextRecord());
    }

    FileUtils.deleteFile(journalFile.getParentFile(), true);
}
 
Example 19
@Test
public void testContentClaimRemoved() throws IOException {
    final File journalFile = new File("target/storage/" + UUID.randomUUID().toString() + "/testSimpleWrite.gz");
    final File tocFile = TocUtil.getTocFile(journalFile);
    final TocWriter tocWriter = new StandardTocWriter(tocFile, false, false);
    final RecordWriter writer = createWriter(journalFile, tocWriter, true, 8192);

    final Map<String, String> attributes = new HashMap<>();
    attributes.put("filename", "1.txt");
    attributes.put("uuid", UUID.randomUUID().toString());

    final ProvenanceEventBuilder builder = new StandardProvenanceEventRecord.Builder();
    builder.setEventTime(System.currentTimeMillis());
    builder.setEventType(ProvenanceEventType.RECEIVE);
    builder.setTransitUri("nifi://unit-test");
    builder.fromFlowFile(TestUtil.createFlowFile(3L, 3000L, attributes));
    builder.setComponentId("1234");
    builder.setComponentType("dummy processor");
    builder.setPreviousContentClaim("container-1", "section-1", "identifier-1", 1L, 1L);
    builder.setCurrentContentClaim(null, null, null, 0L, 0L);
    final ProvenanceEventRecord record = builder.build();

    writer.writeHeader(1L);
    writer.writeRecord(record);
    writer.close();

    final TocReader tocReader = new StandardTocReader(tocFile);

    try (final FileInputStream fis = new FileInputStream(journalFile);
        final RecordReader reader = createReader(fis, journalFile.getName(), tocReader, 2048)) {
        assertEquals(0, reader.getBlockIndex());
        reader.skipToBlock(0);
        final StandardProvenanceEventRecord recovered = reader.nextRecord();
        assertNotNull(recovered);

        assertEquals("nifi://unit-test", recovered.getTransitUri());

        assertEquals("container-1", recovered.getPreviousContentClaimContainer());
        assertNull(recovered.getContentClaimContainer());

        assertEquals("section-1", recovered.getPreviousContentClaimSection());
        assertNull(recovered.getContentClaimSection());

        assertEquals("identifier-1", recovered.getPreviousContentClaimIdentifier());
        assertNull(recovered.getContentClaimIdentifier());

        assertEquals(1L, recovered.getPreviousContentClaimOffset().longValue());
        assertNull(recovered.getContentClaimOffset());

        assertEquals(1L, recovered.getPreviousFileSize().longValue());
        assertEquals(0L, recovered.getFileSize());

        assertNull(reader.nextRecord());
    }

    FileUtils.deleteFile(journalFile.getParentFile(), true);
}
 
Example 20
@Test
public void testContentClaimAdded() throws IOException {
    final File journalFile = new File("target/storage/" + UUID.randomUUID().toString() + "/testSimpleWrite.gz");
    final File tocFile = TocUtil.getTocFile(journalFile);
    final TocWriter tocWriter = new StandardTocWriter(tocFile, false, false);
    final RecordWriter writer = createWriter(journalFile, tocWriter, true, 8192);

    final Map<String, String> attributes = new HashMap<>();
    attributes.put("filename", "1.txt");
    attributes.put("uuid", UUID.randomUUID().toString());

    final ProvenanceEventBuilder builder = new StandardProvenanceEventRecord.Builder();
    builder.setEventTime(System.currentTimeMillis());
    builder.setEventType(ProvenanceEventType.RECEIVE);
    builder.setTransitUri("nifi://unit-test");
    builder.fromFlowFile(TestUtil.createFlowFile(3L, 3000L, attributes));
    builder.setComponentId("1234");
    builder.setComponentType("dummy processor");
    builder.setCurrentContentClaim("container-1", "section-1", "identifier-1", 1L, 1L);
    final ProvenanceEventRecord record = builder.build();

    writer.writeHeader(1L);
    writer.writeRecord(record);
    writer.close();

    final TocReader tocReader = new StandardTocReader(tocFile);

    try (final FileInputStream fis = new FileInputStream(journalFile);
        final RecordReader reader = createReader(fis, journalFile.getName(), tocReader, 2048)) {
        assertEquals(0, reader.getBlockIndex());
        reader.skipToBlock(0);
        final StandardProvenanceEventRecord recovered = reader.nextRecord();
        assertNotNull(recovered);

        assertEquals("nifi://unit-test", recovered.getTransitUri());

        assertEquals("container-1", recovered.getContentClaimContainer());
        assertNull(recovered.getPreviousContentClaimContainer());

        assertEquals("section-1", recovered.getContentClaimSection());
        assertNull(recovered.getPreviousContentClaimSection());

        assertEquals("identifier-1", recovered.getContentClaimIdentifier());
        assertNull(recovered.getPreviousContentClaimIdentifier());

        assertEquals(1L, recovered.getContentClaimOffset().longValue());
        assertNull(recovered.getPreviousContentClaimOffset());

        assertEquals(1L, recovered.getFileSize());
        assertNull(recovered.getPreviousContentClaimOffset());

        assertNull(reader.nextRecord());
    }

    FileUtils.deleteFile(journalFile.getParentFile(), true);
}
 
Example 21
@Test
public void testContentClaimChanged() throws IOException {
    final File journalFile = new File("target/storage/" + UUID.randomUUID().toString() + "/testSimpleWrite.gz");
    final File tocFile = TocUtil.getTocFile(journalFile);
    final TocWriter tocWriter = new StandardTocWriter(tocFile, false, false);
    final RecordWriter writer = createWriter(journalFile, tocWriter, true, 8192);

    final Map<String, String> attributes = new HashMap<>();
    attributes.put("filename", "1.txt");
    attributes.put("uuid", UUID.randomUUID().toString());

    final ProvenanceEventBuilder builder = new StandardProvenanceEventRecord.Builder();
    builder.setEventTime(System.currentTimeMillis());
    builder.setEventType(ProvenanceEventType.RECEIVE);
    builder.setTransitUri("nifi://unit-test");
    builder.fromFlowFile(TestUtil.createFlowFile(3L, 3000L, attributes));
    builder.setComponentId("1234");
    builder.setComponentType("dummy processor");
    builder.setPreviousContentClaim("container-1", "section-1", "identifier-1", 1L, 1L);
    builder.setCurrentContentClaim("container-2", "section-2", "identifier-2", 2L, 2L);
    final ProvenanceEventRecord record = builder.build();

    writer.writeHeader(1L);
    writer.writeRecord(record);
    writer.close();

    final TocReader tocReader = new StandardTocReader(tocFile);

    try (final FileInputStream fis = new FileInputStream(journalFile);
        final RecordReader reader = createReader(fis, journalFile.getName(), tocReader, 2048)) {
        assertEquals(0, reader.getBlockIndex());
        reader.skipToBlock(0);
        final StandardProvenanceEventRecord recovered = reader.nextRecord();
        assertNotNull(recovered);

        assertEquals("nifi://unit-test", recovered.getTransitUri());

        assertEquals("container-1", recovered.getPreviousContentClaimContainer());
        assertEquals("container-2", recovered.getContentClaimContainer());

        assertEquals("section-1", recovered.getPreviousContentClaimSection());
        assertEquals("section-2", recovered.getContentClaimSection());

        assertEquals("identifier-1", recovered.getPreviousContentClaimIdentifier());
        assertEquals("identifier-2", recovered.getContentClaimIdentifier());

        assertEquals(1L, recovered.getPreviousContentClaimOffset().longValue());
        assertEquals(2L, recovered.getContentClaimOffset().longValue());

        assertEquals(1L, recovered.getPreviousFileSize().longValue());
        assertEquals(2L, recovered.getContentClaimOffset().longValue());

        assertNull(reader.nextRecord());
    }

    FileUtils.deleteFile(journalFile.getParentFile(), true);
}
 
Example 22
@Before
public void setup() {
    journalFile = new File("target/storage/" + UUID.randomUUID().toString() + "/testFieldAddedToSchema");
    tocFile = TocUtil.getTocFile(journalFile);
    idGenerator.set(0L);
}
 
Example 23
@Test
@SuppressWarnings("unchecked")
public void testReindex() throws IOException {
    final RepositoryConfiguration repoConfig = createConfig(1, "testReindex");
    repoConfig.setMaxEventFileCount(5);

    final String partitionName = repoConfig.getStorageDirectories().keySet().iterator().next();
    final File storageDirectory = repoConfig.getStorageDirectories().values().iterator().next();

    final RecordWriterFactory recordWriterFactory = (file, idGenerator, compressed, createToc) -> {
        final TocWriter tocWriter = createToc ? new StandardTocWriter(TocUtil.getTocFile(file), false, false) : null;
        return new EventIdFirstSchemaRecordWriter(file, idGenerator, tocWriter, compressed, 32 * 1024, IdentifierLookup.EMPTY);
    };

    final RecordReaderFactory recordReaderFactory = (file, logs, maxChars) -> RecordReaders.newRecordReader(file, logs, maxChars);

    final WriteAheadStorePartition partition = new WriteAheadStorePartition(storageDirectory, partitionName, repoConfig, recordWriterFactory,
        recordReaderFactory, new LinkedBlockingQueue<>(), new AtomicLong(0L), EventReporter.NO_OP);

    for (int i = 0; i < 100; i++) {
        partition.addEvents(Collections.singleton(TestUtil.createEvent()));
    }

    final Map<ProvenanceEventRecord, StorageSummary> reindexedEvents = new HashMap<>();
    final EventIndex eventIndex = Mockito.mock(EventIndex.class);
    Mockito.doAnswer(new Answer<Object>() {
        @Override
        public Object answer(final InvocationOnMock invocation) throws Throwable {
            final Map<ProvenanceEventRecord, StorageSummary> events = invocation.getArgumentAt(0, Map.class);
            reindexedEvents.putAll(events);
            return null;
        }
    }).when(eventIndex).reindexEvents(Mockito.anyMap());

    Mockito.doReturn(18L).when(eventIndex).getMinimumEventIdToReindex("1");
    partition.reindexLatestEvents(eventIndex);

    final List<Long> eventIdsReindexed = reindexedEvents.values().stream()
        .map(StorageSummary::getEventId)
        .sorted()
        .collect(Collectors.toList());

    assertEquals(82, eventIdsReindexed.size());
    for (int i = 0; i < eventIdsReindexed.size(); i++) {
        assertEquals(18 + i, eventIdsReindexed.get(i).intValue());
    }
}
 
Example 24
Source Project: nifi   Source File: RecordWriters.java    License: Apache License 2.0 4 votes vote down vote up
public static RecordWriter newSchemaRecordWriter(final File file, final AtomicLong idGenerator, final boolean compressed, final boolean createToc,
    final int compressionBlockBytes) throws IOException {
    final TocWriter tocWriter = createToc ? new StandardTocWriter(TocUtil.getTocFile(file), false, false) : null;
    return new ByteArraySchemaRecordWriter(file, idGenerator, tocWriter, compressed, compressionBlockBytes);
}
 
Example 25
@Before
public void setup() {
    journalFile = new File("target/storage/" + UUID.randomUUID().toString() + "/testEventIdFirstSchemaRecordReaderWriter");
    tocFile = TocUtil.getTocFile(journalFile);
    idGenerator.set(0L);
}
 
Example 26
@Test
public void testContentClaimUnchanged() throws IOException {
    final File journalFile = new File("target/storage/" + UUID.randomUUID().toString() + "/testSimpleWrite.gz");
    final File tocFile = TocUtil.getTocFile(journalFile);
    final TocWriter tocWriter = new StandardTocWriter(tocFile, false, false);
    final RecordWriter writer = createWriter(journalFile, tocWriter, true, 8192);

    final Map<String, String> attributes = new HashMap<>();
    attributes.put("filename", "1.txt");
    attributes.put("uuid", UUID.randomUUID().toString());

    final ProvenanceEventBuilder builder = new StandardProvenanceEventRecord.Builder();
    builder.setEventTime(System.currentTimeMillis());
    builder.setEventType(ProvenanceEventType.RECEIVE);
    builder.setTransitUri("nifi://unit-test");
    builder.fromFlowFile(TestUtil.createFlowFile(3L, 3000L, attributes));
    builder.setComponentId("1234");
    builder.setComponentType("dummy processor");
    builder.setPreviousContentClaim("container-1", "section-1", "identifier-1", 1L, 1L);
    builder.setCurrentContentClaim("container-1", "section-1", "identifier-1", 1L, 1L);
    final ProvenanceEventRecord record = builder.build();

    writer.writeHeader(1L);
    writer.writeRecord(record);
    writer.close();

    final TocReader tocReader = new StandardTocReader(tocFile);

    try (final FileInputStream fis = new FileInputStream(journalFile);
        final RecordReader reader = createReader(fis, journalFile.getName(), tocReader, 2048)) {
        assertEquals(0, reader.getBlockIndex());
        reader.skipToBlock(0);
        final StandardProvenanceEventRecord recovered = reader.nextRecord();
        assertNotNull(recovered);

        assertEquals("nifi://unit-test", recovered.getTransitUri());

        assertEquals("container-1", recovered.getPreviousContentClaimContainer());
        assertEquals("container-1", recovered.getContentClaimContainer());

        assertEquals("section-1", recovered.getPreviousContentClaimSection());
        assertEquals("section-1", recovered.getContentClaimSection());

        assertEquals("identifier-1", recovered.getPreviousContentClaimIdentifier());
        assertEquals("identifier-1", recovered.getContentClaimIdentifier());

        assertEquals(1L, recovered.getPreviousContentClaimOffset().longValue());
        assertEquals(1L, recovered.getContentClaimOffset().longValue());

        assertEquals(1L, recovered.getPreviousFileSize().longValue());
        assertEquals(1L, recovered.getContentClaimOffset().longValue());

        assertNull(reader.nextRecord());
    }

    FileUtils.deleteFile(journalFile.getParentFile(), true);
}
 
Example 27
@Test
public void testContentClaimRemoved() throws IOException {
    final File journalFile = new File("target/storage/" + UUID.randomUUID().toString() + "/testSimpleWrite.gz");
    final File tocFile = TocUtil.getTocFile(journalFile);
    final TocWriter tocWriter = new StandardTocWriter(tocFile, false, false);
    final RecordWriter writer = createWriter(journalFile, tocWriter, true, 8192);

    final Map<String, String> attributes = new HashMap<>();
    attributes.put("filename", "1.txt");
    attributes.put("uuid", UUID.randomUUID().toString());

    final ProvenanceEventBuilder builder = new StandardProvenanceEventRecord.Builder();
    builder.setEventTime(System.currentTimeMillis());
    builder.setEventType(ProvenanceEventType.RECEIVE);
    builder.setTransitUri("nifi://unit-test");
    builder.fromFlowFile(TestUtil.createFlowFile(3L, 3000L, attributes));
    builder.setComponentId("1234");
    builder.setComponentType("dummy processor");
    builder.setPreviousContentClaim("container-1", "section-1", "identifier-1", 1L, 1L);
    builder.setCurrentContentClaim(null, null, null, 0L, 0L);
    final ProvenanceEventRecord record = builder.build();

    writer.writeHeader(1L);
    writer.writeRecord(record);
    writer.close();

    final TocReader tocReader = new StandardTocReader(tocFile);

    try (final FileInputStream fis = new FileInputStream(journalFile);
        final RecordReader reader = createReader(fis, journalFile.getName(), tocReader, 2048)) {
        assertEquals(0, reader.getBlockIndex());
        reader.skipToBlock(0);
        final StandardProvenanceEventRecord recovered = reader.nextRecord();
        assertNotNull(recovered);

        assertEquals("nifi://unit-test", recovered.getTransitUri());

        assertEquals("container-1", recovered.getPreviousContentClaimContainer());
        assertNull(recovered.getContentClaimContainer());

        assertEquals("section-1", recovered.getPreviousContentClaimSection());
        assertNull(recovered.getContentClaimSection());

        assertEquals("identifier-1", recovered.getPreviousContentClaimIdentifier());
        assertNull(recovered.getContentClaimIdentifier());

        assertEquals(1L, recovered.getPreviousContentClaimOffset().longValue());
        assertNull(recovered.getContentClaimOffset());

        assertEquals(1L, recovered.getPreviousFileSize().longValue());
        assertEquals(0L, recovered.getFileSize());

        assertNull(reader.nextRecord());
    }

    FileUtils.deleteFile(journalFile.getParentFile(), true);
}
 
Example 28
@Test
public void testContentClaimAdded() throws IOException {
    final File journalFile = new File("target/storage/" + UUID.randomUUID().toString() + "/testSimpleWrite.gz");
    final File tocFile = TocUtil.getTocFile(journalFile);
    final TocWriter tocWriter = new StandardTocWriter(tocFile, false, false);
    final RecordWriter writer = createWriter(journalFile, tocWriter, true, 8192);

    final Map<String, String> attributes = new HashMap<>();
    attributes.put("filename", "1.txt");
    attributes.put("uuid", UUID.randomUUID().toString());

    final ProvenanceEventBuilder builder = new StandardProvenanceEventRecord.Builder();
    builder.setEventTime(System.currentTimeMillis());
    builder.setEventType(ProvenanceEventType.RECEIVE);
    builder.setTransitUri("nifi://unit-test");
    builder.fromFlowFile(TestUtil.createFlowFile(3L, 3000L, attributes));
    builder.setComponentId("1234");
    builder.setComponentType("dummy processor");
    builder.setCurrentContentClaim("container-1", "section-1", "identifier-1", 1L, 1L);
    final ProvenanceEventRecord record = builder.build();

    writer.writeHeader(1L);
    writer.writeRecord(record);
    writer.close();

    final TocReader tocReader = new StandardTocReader(tocFile);

    try (final FileInputStream fis = new FileInputStream(journalFile);
        final RecordReader reader = createReader(fis, journalFile.getName(), tocReader, 2048)) {
        assertEquals(0, reader.getBlockIndex());
        reader.skipToBlock(0);
        final StandardProvenanceEventRecord recovered = reader.nextRecord();
        assertNotNull(recovered);

        assertEquals("nifi://unit-test", recovered.getTransitUri());

        assertEquals("container-1", recovered.getContentClaimContainer());
        assertNull(recovered.getPreviousContentClaimContainer());

        assertEquals("section-1", recovered.getContentClaimSection());
        assertNull(recovered.getPreviousContentClaimSection());

        assertEquals("identifier-1", recovered.getContentClaimIdentifier());
        assertNull(recovered.getPreviousContentClaimIdentifier());

        assertEquals(1L, recovered.getContentClaimOffset().longValue());
        assertNull(recovered.getPreviousContentClaimOffset());

        assertEquals(1L, recovered.getFileSize());
        assertNull(recovered.getPreviousContentClaimOffset());

        assertNull(reader.nextRecord());
    }

    FileUtils.deleteFile(journalFile.getParentFile(), true);
}
 
Example 29
@Test
public void testContentClaimChanged() throws IOException {
    final File journalFile = new File("target/storage/" + UUID.randomUUID().toString() + "/testSimpleWrite.gz");
    final File tocFile = TocUtil.getTocFile(journalFile);
    final TocWriter tocWriter = new StandardTocWriter(tocFile, false, false);
    final RecordWriter writer = createWriter(journalFile, tocWriter, true, 8192);

    final Map<String, String> attributes = new HashMap<>();
    attributes.put("filename", "1.txt");
    attributes.put("uuid", UUID.randomUUID().toString());

    final ProvenanceEventBuilder builder = new StandardProvenanceEventRecord.Builder();
    builder.setEventTime(System.currentTimeMillis());
    builder.setEventType(ProvenanceEventType.RECEIVE);
    builder.setTransitUri("nifi://unit-test");
    builder.fromFlowFile(TestUtil.createFlowFile(3L, 3000L, attributes));
    builder.setComponentId("1234");
    builder.setComponentType("dummy processor");
    builder.setPreviousContentClaim("container-1", "section-1", "identifier-1", 1L, 1L);
    builder.setCurrentContentClaim("container-2", "section-2", "identifier-2", 2L, 2L);
    final ProvenanceEventRecord record = builder.build();

    writer.writeHeader(1L);
    writer.writeRecord(record);
    writer.close();

    final TocReader tocReader = new StandardTocReader(tocFile);

    try (final FileInputStream fis = new FileInputStream(journalFile);
        final RecordReader reader = createReader(fis, journalFile.getName(), tocReader, 2048)) {
        assertEquals(0, reader.getBlockIndex());
        reader.skipToBlock(0);
        final StandardProvenanceEventRecord recovered = reader.nextRecord();
        assertNotNull(recovered);

        assertEquals("nifi://unit-test", recovered.getTransitUri());

        assertEquals("container-1", recovered.getPreviousContentClaimContainer());
        assertEquals("container-2", recovered.getContentClaimContainer());

        assertEquals("section-1", recovered.getPreviousContentClaimSection());
        assertEquals("section-2", recovered.getContentClaimSection());

        assertEquals("identifier-1", recovered.getPreviousContentClaimIdentifier());
        assertEquals("identifier-2", recovered.getContentClaimIdentifier());

        assertEquals(1L, recovered.getPreviousContentClaimOffset().longValue());
        assertEquals(2L, recovered.getContentClaimOffset().longValue());

        assertEquals(1L, recovered.getPreviousFileSize().longValue());
        assertEquals(2L, recovered.getContentClaimOffset().longValue());

        assertNull(reader.nextRecord());
    }

    FileUtils.deleteFile(journalFile.getParentFile(), true);
}
 
Example 30
Source Project: nifi   Source File: TestSchemaRecordReaderWriter.java    License: Apache License 2.0 4 votes vote down vote up
@Before
public void setup() {
    journalFile = new File("target/storage/" + UUID.randomUUID().toString() + "/testFieldAddedToSchema");
    tocFile = TocUtil.getTocFile(journalFile);
    idGenerator.set(0L);
}