Java Code Examples for org.apache.nifi.util.FileUtils#deleteFile()

/**
 * Unpacks the specified nar into the specified base working directory.
 *
 * @param nar
 *            the nar to unpack
 * @param baseWorkingDirectory
 *            the directory to unpack to
 * @return the directory to the unpacked NAR
 * @throws IOException
 *             if unable to explode nar
 */
private static File unpackNar(final File nar, final File baseWorkingDirectory)
        throws IOException {
    final File narWorkingDirectory = new File(baseWorkingDirectory, nar.getName() + "-unpacked");

    // if the working directory doesn't exist, unpack the nar
    if (!narWorkingDirectory.exists()) {
        unpack(nar, narWorkingDirectory, calculateMd5sum(nar));
    } else {
        // the working directory does exist. Run MD5 sum against the nar
        // file and check if the nar has changed since it was deployed.
        final byte[] narMd5 = calculateMd5sum(nar);
        final File workingHashFile = new File(narWorkingDirectory, HASH_FILENAME);
        if (!workingHashFile.exists()) {
            FileUtils.deleteFile(narWorkingDirectory, true);
            unpack(nar, narWorkingDirectory, narMd5);
        } else {
            final byte[] hashFileContents = Files.readAllBytes(workingHashFile.toPath());
            if (!Arrays.equals(hashFileContents, narMd5)) {
                logger.info("Contents of nar {} have changed. Reloading.",
                        new Object[] { nar.getAbsolutePath() });
                FileUtils.deleteFile(narWorkingDirectory, true);
                unpack(nar, narWorkingDirectory, narMd5);
            }
        }
    }

    return narWorkingDirectory;
}
 

/**
 * Unpacks the specified nar into the specified base working directory.
 *
 * @param nar
 *            the nar to unpack
 * @param baseWorkingDirectory
 *            the directory to unpack to
 * @return the directory to the unpacked NAR
 * @throws IOException
 *             if unable to explode nar
 */
private static File unpackNar(final File nar, final File baseWorkingDirectory)
        throws IOException {
    final File narWorkingDirectory = new File(baseWorkingDirectory, nar.getName() + "-unpacked");

    // if the working directory doesn't exist, unpack the nar
    if (!narWorkingDirectory.exists()) {
        unpack(nar, narWorkingDirectory, calculateMd5sum(nar));
    } else {
        // the working directory does exist. Run MD5 sum against the nar
        // file and check if the nar has changed since it was deployed.
        final byte[] narMd5 = calculateMd5sum(nar);
        final File workingHashFile = new File(narWorkingDirectory, HASH_FILENAME);
        if (!workingHashFile.exists()) {
            FileUtils.deleteFile(narWorkingDirectory, true);
            unpack(nar, narWorkingDirectory, narMd5);
        } else {
            final byte[] hashFileContents = Files.readAllBytes(workingHashFile.toPath());
            if (!Arrays.equals(hashFileContents, narMd5)) {
                logger.info("Contents of nar {} have changed. Reloading.",
                        new Object[] { nar.getAbsolutePath() });
                FileUtils.deleteFile(narWorkingDirectory, true);
                unpack(nar, narWorkingDirectory, narMd5);
            }
        }
    }

    return narWorkingDirectory;
}
 

private void deleteDirectory(final File dir) throws IOException {
    if (!dir.exists()) {
        return;
    }

    FileUtils.deleteFile(dir, true);
}
 

/**
 * Unpacks the specified nar into the specified base working directory.
 *
 * @param nar the nar to unpack
 * @param baseWorkingDirectory the directory to unpack to
 * @return the directory to the unpacked NAR
 * @throws IOException if unable to explode nar
 */
public static File unpackNar(final File nar, final File baseWorkingDirectory) throws IOException {
    final File narWorkingDirectory = new File(baseWorkingDirectory, nar.getName() + "-unpacked");

    // if the working directory doesn't exist, unpack the nar
    if (!narWorkingDirectory.exists()) {
        unpack(nar, narWorkingDirectory, calculateMd5sum(nar));
    } else {
        // the working directory does exist. Run MD5 sum against the nar
        // file and check if the nar has changed since it was deployed.
        final byte[] narMd5 = calculateMd5sum(nar);
        final File workingHashFile = new File(narWorkingDirectory, HASH_FILENAME);
        if (!workingHashFile.exists()) {
            FileUtils.deleteFile(narWorkingDirectory, true);
            unpack(nar, narWorkingDirectory, narMd5);
        } else {
            final byte[] hashFileContents = Files.readAllBytes(workingHashFile.toPath());
            if (!Arrays.equals(hashFileContents, narMd5)) {
                logger.info("Contents of nar {} have changed. Reloading.", new Object[] { nar.getAbsolutePath() });
                FileUtils.deleteFile(narWorkingDirectory, true);
                unpack(nar, narWorkingDirectory, narMd5);
            }
        }
    }

    return narWorkingDirectory;
}
 

public NiFi(final NiFiProperties properties, ClassLoader rootClassLoader)
        throws ClassNotFoundException, IOException, NoSuchMethodException, InstantiationException, IllegalAccessException, IllegalArgumentException, InvocationTargetException {

    // There can only be one krb5.conf for the overall Java process so set this globally during
    // start up so that processors and our Kerberos authentication code don't have to set this
    final File kerberosConfigFile = properties.getKerberosConfigurationFile();
    if (kerberosConfigFile != null) {
        final String kerberosConfigFilePath = kerberosConfigFile.getAbsolutePath();
        LOGGER.info("Setting java.security.krb5.conf to {}", new Object[]{kerberosConfigFilePath});
        System.setProperty("java.security.krb5.conf", kerberosConfigFilePath);
    }

    setDefaultUncaughtExceptionHandler();

    // register the shutdown hook
    addShutdownHook();

    final String bootstrapPort = System.getProperty(BOOTSTRAP_PORT_PROPERTY);
    if (bootstrapPort != null) {
        try {
            final int port = Integer.parseInt(bootstrapPort);

            if (port < 1 || port > 65535) {
                throw new RuntimeException("Failed to start NiFi because system property '" + BOOTSTRAP_PORT_PROPERTY + "' is not a valid integer in the range 1 - 65535");
            }

            bootstrapListener = new BootstrapListener(this, port);
            bootstrapListener.start();
        } catch (final NumberFormatException nfe) {
            throw new RuntimeException("Failed to start NiFi because system property '" + BOOTSTRAP_PORT_PROPERTY + "' is not a valid integer in the range 1 - 65535");
        }
    } else {
        LOGGER.info("NiFi started without Bootstrap Port information provided; will not listen for requests from Bootstrap");
        bootstrapListener = null;
    }

    // delete the web working dir - if the application does not start successfully
    // the web app directories might be in an invalid state. when this happens
    // jetty will not attempt to re-extract the war into the directory. by removing
    // the working directory, we can be assured that it will attempt to extract the
    // war every time the application starts.
    File webWorkingDir = properties.getWebWorkingDirectory();
    FileUtils.deleteFilesInDirectory(webWorkingDir, null, LOGGER, true, true);
    FileUtils.deleteFile(webWorkingDir, LOGGER, 3);

    detectTimingIssues();

    // redirect JUL log events
    initLogging();

    final Bundle systemBundle = SystemBundle.create(properties, rootClassLoader);

    // expand the nars
    final ExtensionMapping extensionMapping = NarUnpacker.unpackNars(properties, systemBundle);

    // load the extensions classloaders
    NarClassLoaders narClassLoaders = NarClassLoadersHolder.getInstance();

    narClassLoaders.init(rootClassLoader,
            properties.getFrameworkWorkingDirectory(), properties.getExtensionsWorkingDirectory());

    // load the framework classloader
    final ClassLoader frameworkClassLoader = narClassLoaders.getFrameworkBundle().getClassLoader();
    if (frameworkClassLoader == null) {
        throw new IllegalStateException("Unable to find the framework NAR ClassLoader.");
    }

    final Set<Bundle> narBundles = narClassLoaders.getBundles();

    // load the server from the framework classloader
    Thread.currentThread().setContextClassLoader(frameworkClassLoader);
    Class<?> jettyServer = Class.forName("org.apache.nifi.web.server.JettyServer", true, frameworkClassLoader);
    Constructor<?> jettyConstructor = jettyServer.getConstructor(NiFiProperties.class, Set.class);

    final long startTime = System.nanoTime();
    nifiServer = (NiFiServer) jettyConstructor.newInstance(properties, narBundles);
    nifiServer.setExtensionMapping(extensionMapping);
    nifiServer.setBundles(systemBundle, narBundles);

    if (shutdown) {
        LOGGER.info("NiFi has been shutdown via NiFi Bootstrap. Will not start Controller");
    } else {
        nifiServer.start();

        if (bootstrapListener != null) {
            bootstrapListener.sendStartedStatus(true);
        }

        final long duration = System.nanoTime() - startTime;
        LOGGER.info("Controller initialization took " + duration + " nanoseconds "
                + "(" + (int) TimeUnit.SECONDS.convert(duration, TimeUnit.NANOSECONDS) + " seconds).");
    }
}
 

protected final void initialize(final NiFiProperties nifiProperties) throws IOException {
    this.nifiProperties = nifiProperties;

    final FlowFileEventRepository flowFileEventRepository = new RingBufferEventRepository(5);

    final BulletinRepository bulletinRepo = new VolatileBulletinRepository();
    flowEngine = new FlowEngine(4, "unit test flow engine");
    extensionManager = new DirectInjectionExtensionManager();

    extensionManager.injectExtensionType(FlowFileRepository.class, WriteAheadFlowFileRepository.class);
    extensionManager.injectExtensionType(ContentRepository.class, FileSystemRepository.class);
    extensionManager.injectExtensionType(ProvenanceRepository.class, WriteAheadProvenanceRepository.class);
    extensionManager.injectExtensionType(StateProvider.class, WriteAheadLocalStateProvider.class);
    extensionManager.injectExtensionType(ComponentStatusRepository.class, VolatileComponentStatusRepository.class);
    extensionManager.injectExtensionType(FlowFileSwapManager.class, FileSystemSwapManager.class);

    extensionManager.injectExtensionType(Processor.class, BiConsumerProcessor.class);
    extensionManager.injectExtensionType(Processor.class, GenerateProcessor.class);
    extensionManager.injectExtensionType(Processor.class, TerminateOnce.class);
    extensionManager.injectExtensionType(Processor.class, TerminateAll.class);
    extensionManager.injectExtensionType(Processor.class, NopProcessor.class);
    extensionManager.injectExtensionType(Processor.class, UsernamePasswordProcessor.class);

    injectExtensionTypes(extensionManager);
    systemBundle = SystemBundle.create(nifiProperties);
    extensionManager.discoverExtensions(systemBundle, Collections.emptySet());

    final StringEncryptor encryptor = StringEncryptor.createEncryptor("PBEWITHMD5AND256BITAES-CBC-OPENSSL", "BC", "unit-test");
    final Authorizer authorizer = new AlwaysAuthorizedAuthorizer();
    final AuditService auditService = new NopAuditService();

    if (isClusteredTest()) {
        final File zookeeperDir = new File("target/state/zookeeper");
        final File version2Dir =  new File(zookeeperDir, "version-2");

        if (!version2Dir.exists()) {
            assertTrue(version2Dir.mkdirs());
        }

        final File[] children = version2Dir.listFiles();
        if (children != null) {
            for (final File file : children) {
                FileUtils.deleteFile(file, true);
            }
        }

        clusterCoordinator = Mockito.mock(ClusterCoordinator.class);
        final HeartbeatMonitor heartbeatMonitor = Mockito.mock(HeartbeatMonitor.class);
        final NodeProtocolSender protocolSender = Mockito.mock(NodeProtocolSender.class);
        final LeaderElectionManager leaderElectionManager = new CuratorLeaderElectionManager(2, nifiProperties);

        final NodeIdentifier localNodeId = new NodeIdentifier(UUID.randomUUID().toString(), "localhost", 8111, "localhost", 8081,
            "localhost", 8082, "localhost", 8083, 8084, false, Collections.emptySet());
        final NodeIdentifier node2Id = new NodeIdentifier(UUID.randomUUID().toString(), "localhost", 8222, "localhost", 8081,
            "localhost", 8082, "localhost", 8083, 8084, false, Collections.emptySet());

        final Set<NodeIdentifier> nodeIdentifiers = new HashSet<>();
        nodeIdentifiers.add(localNodeId);
        nodeIdentifiers.add(node2Id);
        Mockito.when(clusterCoordinator.getNodeIdentifiers()).thenReturn(nodeIdentifiers);
        Mockito.when(clusterCoordinator.getLocalNodeIdentifier()).thenReturn(localNodeId);

        flowController = FlowController.createClusteredInstance(flowFileEventRepository, nifiProperties, authorizer, auditService, encryptor, protocolSender, bulletinRepo, clusterCoordinator,
            heartbeatMonitor, leaderElectionManager, VariableRegistry.ENVIRONMENT_SYSTEM_REGISTRY, flowRegistryClient, extensionManager);

        flowController.setClustered(true, UUID.randomUUID().toString());
        flowController.setNodeId(localNodeId);

        flowController.setConnectionStatus(new NodeConnectionStatus(localNodeId, NodeConnectionState.CONNECTED));
    } else {
        flowController = FlowController.createStandaloneInstance(flowFileEventRepository, nifiProperties, authorizer, auditService, encryptor, bulletinRepo,
            VariableRegistry.ENVIRONMENT_SYSTEM_REGISTRY, flowRegistryClient, extensionManager);
    }

    processScheduler = new StandardProcessScheduler(flowEngine, flowController, encryptor, flowController.getStateManagerProvider(), nifiProperties);

    final RepositoryContextFactory repositoryContextFactory = flowController.getRepositoryContextFactory();
    final SchedulingAgent timerDrivenSchedulingAgent = new TimerDrivenSchedulingAgent(flowController, flowEngine, repositoryContextFactory, encryptor, nifiProperties);
    processScheduler.setSchedulingAgent(SchedulingStrategy.TIMER_DRIVEN, timerDrivenSchedulingAgent);

    flowFileSwapManager = flowController.createSwapManager();
    resourceClaimManager = flowController.getResourceClaimManager();
}