org.elasticsearch.common.io.FileSystemUtils Java Examples

The following examples show how to use org.elasticsearch.common.io.FileSystemUtils. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: InternalTestCluster.java    From crate with Apache License 2.0 6 votes vote down vote up
public synchronized void wipePendingDataDirectories() {
    if (!dataDirToClean.isEmpty()) {
        try {
            for (Path path : dataDirToClean) {
                try {
                    FileSystemUtils.deleteSubDirectories(path);
                    logger.info("Successfully wiped data directory for node location: {}", path);
                } catch (IOException e) {
                    logger.info("Failed to wipe data directory for node location: {}", path);
                }
            }
        } finally {
            dataDirToClean.clear();
        }
    }
}
 
Example #2
Source File: PluginsService.java    From crate with Apache License 2.0 6 votes vote down vote up
/**
 * Extracts all installed plugin directories from the provided {@code rootPath}.
 *
 * @param rootPath the path where the plugins are installed
 * @return a list of all plugin paths installed in the {@code rootPath}
 * @throws IOException if an I/O exception occurred reading the directories
 */
public static List<Path> findPluginDirs(final Path rootPath) throws IOException {
    final List<Path> plugins = new ArrayList<>();
    final Set<String> seen = new HashSet<>();
    if (Files.exists(rootPath)) {
        try (DirectoryStream<Path> stream = Files.newDirectoryStream(rootPath)) {
            for (Path plugin : stream) {
                if (FileSystemUtils.isHidden(plugin)) {
                    continue;
                }
                if (FileSystemUtils.isDesktopServicesStore(plugin) ||
                    plugin.getFileName().toString().startsWith(".removing-")) {
                    continue;
                }
                if (seen.add(plugin.getFileName().toString()) == false) {
                    throw new IllegalStateException("duplicate plugin: " + plugin);
                } else if (Files.exists(plugin.resolve(ES_PLUGIN_PROPERTIES))) {
                    plugins.add(plugin);
                }
            }
        }
    }
    return plugins;
}
 
Example #3
Source File: Analysis.java    From Elasticsearch with Apache License 2.0 6 votes vote down vote up
/**
 * Fetches a list of words from the specified settings file. The list should either be available at the key
 * specified by settingsPrefix or in a file specified by settingsPrefix + _path.
 *
 * @throws IllegalArgumentException
 *          If the word list cannot be found at either key.
 */
public static List<String> getWordList(Environment env, Settings settings, String settingPrefix) {
    String wordListPath = settings.get(settingPrefix + "_path", null);

    if (wordListPath == null) {
        String[] explicitWordList = settings.getAsArray(settingPrefix, null);
        if (explicitWordList == null) {
            return null;
        } else {
            return Arrays.asList(explicitWordList);
        }
    }

    final Path wordListFile = env.configFile().resolve(wordListPath);

    try (BufferedReader reader = FileSystemUtils.newBufferedReader(wordListFile.toUri().toURL(), Charsets.UTF_8)) {
        return loadWordList(reader, "#");
    } catch (IOException ioe) {
        String message = String.format(Locale.ROOT, "IOException while reading %s_path: %s", settingPrefix, ioe.getMessage());
        throw new IllegalArgumentException(message);
    }
}
 
Example #4
Source File: Analysis.java    From Elasticsearch with Apache License 2.0 6 votes vote down vote up
/**
 * @return null If no settings set for "settingsPrefix" then return <code>null</code>.
 * @throws IllegalArgumentException
 *          If the Reader can not be instantiated.
 */
public static Reader getReaderFromFile(Environment env, Settings settings, String settingPrefix) {
    String filePath = settings.get(settingPrefix, null);

    if (filePath == null) {
        return null;
    }

    final Path path = env.configFile().resolve(filePath);

    try {
        return FileSystemUtils.newBufferedReader(path.toUri().toURL(), Charsets.UTF_8);
    } catch (IOException ioe) {
        String message = String.format(Locale.ROOT, "IOException while reading %s_path: %s", settingPrefix, ioe.getMessage());
        throw new IllegalArgumentException(message);
    }
}
 
Example #5
Source File: MetaDataCreateIndexService.java    From Elasticsearch with Apache License 2.0 6 votes vote down vote up
private void addMappings(Map<String, Map<String, Object>> mappings, Path mappingsDir) throws IOException {
    try (DirectoryStream<Path> stream = Files.newDirectoryStream(mappingsDir)) {
        for (Path mappingFile : stream) {
            final String fileName = mappingFile.getFileName().toString();
            if (FileSystemUtils.isHidden(mappingFile)) {
                continue;
            }
            int lastDotIndex = fileName.lastIndexOf('.');
            String mappingType = lastDotIndex != -1 ? mappingFile.getFileName().toString().substring(0, lastDotIndex) : mappingFile.getFileName().toString();
            try (BufferedReader reader = Files.newBufferedReader(mappingFile, Charsets.UTF_8)) {
                String mappingSource = Streams.copyToString(reader);
                if (mappings.containsKey(mappingType)) {
                    XContentHelper.mergeDefaults(mappings.get(mappingType), parseMapping(mappingSource));
                } else {
                    mappings.put(mappingType, parseMapping(mappingSource));
                }
            } catch (Exception e) {
                logger.warn("failed to read / parse mapping [" + mappingType + "] from location [" + mappingFile + "], ignoring...", e);
            }
        }
    }
}
 
Example #6
Source File: IndicesService.java    From Elasticsearch with Apache License 2.0 6 votes vote down vote up
private boolean canDeleteShardContent(ShardId shardId, Settings indexSettings) {
    final IndexServiceInjectorPair indexServiceInjectorPair = this.indices.get(shardId.getIndex());
    if (IndexMetaData.isOnSharedFilesystem(indexSettings) == false) {
         if (nodeEnv.hasNodeFile()) {
            boolean isAllocated = indexServiceInjectorPair != null && indexServiceInjectorPair
                .getIndexService().hasShard(shardId.getId());
            if (isAllocated) {
                return false; // we are allocated - can't delete the shard
            }
            if (NodeEnvironment.hasCustomDataPath(indexSettings)) {
                // lets see if it's on a custom path (return false if the shared doesn't exist)
                // we don't need to delete anything that is not there
                return Files.exists(nodeEnv.resolveCustomLocation(indexSettings, shardId));
            } else {
                // lets see if it's path is available (return false if the shared doesn't exist)
                // we don't need to delete anything that is not there
                return FileSystemUtils.exists(nodeEnv.availableShardPaths(shardId));
            }
        }
    } else {
        logger.trace("{} skipping shard directory deletion due to shadow replicas", shardId);
    }
    return false;
}
 
Example #7
Source File: PluginManager.java    From Elasticsearch with Apache License 2.0 5 votes vote down vote up
/** we check whether we need to remove the top-level folder while extracting
 *  sometimes (e.g. github) the downloaded archive contains a top-level folder which needs to be removed
 */
private Path findPluginRoot(Path dir) throws IOException {
    if (Files.exists(dir.resolve(PluginInfo.ES_PLUGIN_PROPERTIES))) {
        return dir;
    } else {
        final Path[] topLevelFiles = FileSystemUtils.files(dir);
        if (topLevelFiles.length == 1 && Files.isDirectory(topLevelFiles[0])) {
            Path subdir = topLevelFiles[0];
            if (Files.exists(subdir.resolve(PluginInfo.ES_PLUGIN_PROPERTIES))) {
                return subdir;
            }
        }
    }
    throw new RuntimeException("Could not find plugin descriptor '" + PluginInfo.ES_PLUGIN_PROPERTIES + "' in plugin zip");
}
 
Example #8
Source File: PluginManager.java    From Elasticsearch with Apache License 2.0 5 votes vote down vote up
/** check a candidate plugin for jar hell before installing it */
private void jarHellCheck(Path candidate, boolean isolated) throws IOException {
    // create list of current jars in classpath
    final List<URL> jars = new ArrayList<>();
    jars.addAll(Arrays.asList(JarHell.parseClassPath()));

    // read existing bundles. this does some checks on the installation too.
    List<Bundle> bundles = PluginsService.getPluginBundles(environment.pluginsFile());

    // if we aren't isolated, we need to jarhellcheck against any other non-isolated plugins
    // thats always the first bundle
    if (isolated == false) {
        jars.addAll(bundles.get(0).urls);
    }

    // add plugin jars to the list
    Path pluginJars[] = FileSystemUtils.files(candidate, "*.jar");
    for (Path jar : pluginJars) {
        jars.add(jar.toUri().toURL());
    }

    // check combined (current classpath + new jars to-be-added)
    try {
        JarHell.checkJarHell(jars.toArray(new URL[jars.size()]));
    } catch (Exception ex) {
        throw new RuntimeException(ex);
    }
}
 
Example #9
Source File: PluginsService.java    From Elasticsearch with Apache License 2.0 5 votes vote down vote up
static List<Bundle> getModuleBundles(Path modulesDirectory) throws IOException {
    // damn leniency
    if (Files.notExists(modulesDirectory)) {
        return Collections.emptyList();
    }
    List<Bundle> bundles = new ArrayList<>();
    try (DirectoryStream<Path> stream = Files.newDirectoryStream(modulesDirectory)) {
        for (Path module : stream) {
            if (FileSystemUtils.isHidden(module)) {
                continue; // skip over .DS_Store etc
            }
            PluginInfo info = PluginInfo.readFromProperties(module);
            if (!info.isJvm()) {
                throw new IllegalStateException("modules must be jvm plugins: " + info);
            }
            if (!info.isIsolated()) {
                throw new IllegalStateException("modules must be isolated: " + info);
            }
            Bundle bundle = new Bundle();
            bundle.plugins.add(info);
            // gather urls for jar files
            try (DirectoryStream<Path> jarStream = Files.newDirectoryStream(module, "*.jar")) {
                for (Path jar : jarStream) {
                    // normalize with toRealPath to get symlinks out of our hair
                    bundle.urls.add(jar.toRealPath().toUri().toURL());
                }
            }
            bundles.add(bundle);
        }
    }
    return bundles;
}
 
Example #10
Source File: PluginsService.java    From Elasticsearch with Apache License 2.0 4 votes vote down vote up
static List<Bundle> getPluginBundles(Path pluginsDirectory) throws IOException {
    ESLogger logger = Loggers.getLogger(PluginsService.class);

    // TODO: remove this leniency, but tests bogusly rely on it
    if (!isAccessibleDirectory(pluginsDirectory, logger)) {
        return Collections.emptyList();
    }

    List<Bundle> bundles = new ArrayList<>();
    // a special purgatory for plugins that directly depend on each other
    bundles.add(new Bundle());

    try (DirectoryStream<Path> stream = Files.newDirectoryStream(pluginsDirectory)) {
        for (Path plugin : stream) {
            if (FileSystemUtils.isHidden(plugin)) {
                logger.trace("--- skip hidden plugin file[{}]", plugin.toAbsolutePath());
                continue;
            }
            if (!FileSystemUtils.isAccessibleDirectory(plugin, logger)) {
                continue;
            }
            PluginInfo info;
            try {
                info = PluginInfo.readFromProperties(plugin);
            } catch (NoSuchFileException e) {
                // es plugin descriptor file not found, ignore, could be a Crate plugin
                logger.trace("--- plugin descriptor file not found, ignoring plugin [{}]", plugin.toAbsolutePath());
                continue;
            }
            logger.trace("--- adding plugin [{}]", plugin.toAbsolutePath());
            List<URL> urls = new ArrayList<>();
            if (info.isJvm()) {
                // a jvm plugin: gather urls for jar files
                try (DirectoryStream<Path> jarStream = Files.newDirectoryStream(plugin, "*.jar")) {
                    for (Path jar : jarStream) {
                        // normalize with toRealPath to get symlinks out of our hair
                        urls.add(jar.toRealPath().toUri().toURL());
                    }
                }
            }
            final Bundle bundle;
            if (info.isJvm() && info.isIsolated() == false) {
                bundle = bundles.get(0); // purgatory
            } else {
                bundle = new Bundle();
                bundles.add(bundle);
            }
            bundle.plugins.add(info);
            bundle.urls.addAll(urls);
        }
    }

    return bundles;
}
 
Example #11
Source File: Neo4jRiverIntTest.java    From elasticsearch-river-neo4j with Apache License 2.0 4 votes vote down vote up
@Before
public void setupElasticAndNeo4jClient() throws Exception {

    Settings globalSettings = settingsBuilder().loadFromClasspath("settings.yml").build();
    String json = copyToStringFromClasspath("/neo4j-inttest-river.json");
    Settings riverSettings = settingsBuilder().loadFromSource(json).build();

    index = riverSettings.get("index.name");
    type = riverSettings.get("index.type");
    String uri = riverSettings.get("neo4j.uri");

    logger.debug("Connecting to neo4j @ {}", uri);
    db = new SpringCypherRestGraphDatabase(uri);

    logger.debug("Starting local elastic...");
    Tuple<Settings, Environment> initialSettings = InternalSettingsPerparer.prepareSettings(globalSettings, true);

    if (!initialSettings.v2().configFile().exists()) {
        FileSystemUtils.mkdirs(initialSettings.v2().configFile());
    }

    if (!initialSettings.v2().logsFile().exists()) {
        FileSystemUtils.mkdirs(initialSettings.v2().logsFile());
    }

    node = nodeBuilder().local(true).settings(globalSettings).node();

    logger.info("Create river [{}]", river);
    node.client().prepareIndex("_river", river, "_meta").setSource(json).execute().actionGet();

    logger.debug("Running Cluster Health");
    ClusterHealthResponse clusterHealth = node.client().admin().cluster()
            .health(clusterHealthRequest().waitForGreenStatus())
            .actionGet();
    logger.info("Done Cluster Health, status " + clusterHealth.getStatus());

    GetResponse response = node.client().prepareGet("_river", river, "_meta").execute().actionGet();
    assertTrue(response.isExists());

    logger.debug("...elasticized ok");
}
 
Example #12
Source File: ElasticSearchResource.java    From usergrid with Apache License 2.0 4 votes vote down vote up
@Override
public void start(Description description) throws Exception {
    FileSystemUtils.deleteRecursively(new File(embedded.getConfig().getDataDir()));
    embedded.start();
}
 
Example #13
Source File: ElasticSearchResource.java    From usergrid with Apache License 2.0 4 votes vote down vote up
@Override
public void stop(Description description) {
    embedded.stop();
    FileSystemUtils.deleteRecursively(new File(embedded.getConfig().getDataDir()));
}
 
Example #14
Source File: HunspellService.java    From crate with Apache License 2.0 4 votes vote down vote up
/**
 * Loads the hunspell dictionary for the given local.
 *
 * @param locale       The locale of the hunspell dictionary to be loaded.
 * @param nodeSettings The node level settings
 * @param env          The node environment (from which the conf path will be resolved)
 * @return The loaded Hunspell dictionary
 * @throws Exception when loading fails (due to IO errors or malformed dictionary files)
 */
private Dictionary loadDictionary(String locale, Settings nodeSettings, Environment env) throws Exception {
    if (LOGGER.isDebugEnabled()) {
        LOGGER.debug("Loading hunspell dictionary [{}]...", locale);
    }
    Path dicDir = hunspellDir.resolve(locale);
    if (FileSystemUtils.isAccessibleDirectory(dicDir, LOGGER) == false) {
        throw new ElasticsearchException(String.format(Locale.ROOT, "Could not find hunspell dictionary [%s]", locale));
    }

    // merging node settings with hunspell dictionary specific settings
    Settings dictSettings = HUNSPELL_DICTIONARY_OPTIONS.get(nodeSettings);
    nodeSettings = loadDictionarySettings(dicDir, dictSettings.getByPrefix(locale + "."));

    boolean ignoreCase = nodeSettings.getAsBoolean("ignore_case", defaultIgnoreCase);

    Path[] affixFiles = FileSystemUtils.files(dicDir, "*.aff");
    if (affixFiles.length == 0) {
        throw new ElasticsearchException(String.format(Locale.ROOT, "Missing affix file for hunspell dictionary [%s]", locale));
    }
    if (affixFiles.length != 1) {
        throw new ElasticsearchException(String.format(Locale.ROOT, "Too many affix files exist for hunspell dictionary [%s]", locale));
    }
    InputStream affixStream = null;

    Path[] dicFiles = FileSystemUtils.files(dicDir, "*.dic");
    List<InputStream> dicStreams = new ArrayList<>(dicFiles.length);
    try {

        for (int i = 0; i < dicFiles.length; i++) {
            dicStreams.add(Files.newInputStream(dicFiles[i]));
        }

        affixStream = Files.newInputStream(affixFiles[0]);

        try (Directory tmp = new SimpleFSDirectory(env.tmpFile())) {
            return new Dictionary(tmp, "hunspell", affixStream, dicStreams, ignoreCase);
        }

    } catch (Exception e) {
        LOGGER.error(() -> new ParameterizedMessage("Could not load hunspell dictionary [{}]", locale), e);
        throw e;
    } finally {
        IOUtils.close(affixStream);
        IOUtils.close(dicStreams);
    }
}
 
Example #15
Source File: HttpServer.java    From Elasticsearch with Apache License 2.0 4 votes vote down vote up
void handlePluginSite(HttpRequest request, HttpChannel channel) throws IOException {
    if (disableSites) {
        channel.sendResponse(new BytesRestResponse(FORBIDDEN));
        return;
    }
    if (request.method() == RestRequest.Method.OPTIONS) {
        // when we have OPTIONS request, simply send OK by default (with the Access Control Origin header which gets automatically added)
        channel.sendResponse(new BytesRestResponse(OK));
        return;
    }
    if (request.method() != RestRequest.Method.GET) {
        channel.sendResponse(new BytesRestResponse(FORBIDDEN));
        return;
    }
    // TODO for a "/_plugin" endpoint, we should have a page that lists all the plugins?

    String path = request.rawPath().substring("/_plugin/".length());
    int i1 = path.indexOf('/');
    String pluginName;
    String sitePath;
    if (i1 == -1) {
        pluginName = path;
        sitePath = null;
        // If a trailing / is missing, we redirect to the right page #2654
        String redirectUrl = request.rawPath() + "/";
        BytesRestResponse restResponse = new BytesRestResponse(RestStatus.MOVED_PERMANENTLY, "text/html", "<head><meta http-equiv=\"refresh\" content=\"0; URL=" + redirectUrl + "\"></head>");
        restResponse.addHeader("Location", redirectUrl);
        channel.sendResponse(restResponse);
        return;
    } else {
        pluginName = path.substring(0, i1);
        sitePath = path.substring(i1 + 1);
    }

    // we default to index.html, or what the plugin provides (as a unix-style path)
    // this is a relative path under _site configured by the plugin.
    if (sitePath.length() == 0) {
        sitePath = "index.html";
    } else {
        // remove extraneous leading slashes, its not an absolute path.
        while (sitePath.length() > 0 && sitePath.charAt(0) == '/') {
            sitePath = sitePath.substring(1);
        }
    }
    final Path siteFile = environment.pluginsFile().resolve(pluginName).resolve("_site");

    final String separator = siteFile.getFileSystem().getSeparator();
    // Convert file separators.
    sitePath = sitePath.replace("/", separator);
    
    Path file = siteFile.resolve(sitePath);

    // return not found instead of forbidden to prevent malicious requests to find out if files exist or dont exist
    if (!Files.exists(file) || FileSystemUtils.isHidden(file) || !file.toAbsolutePath().normalize().startsWith(siteFile.toAbsolutePath().normalize())) {
        channel.sendResponse(new BytesRestResponse(NOT_FOUND));
        return;
    }

    BasicFileAttributes attributes = Files.readAttributes(file, BasicFileAttributes.class);
    if (!attributes.isRegularFile()) {
        // If it's not a dir, we send a 403
        if (!attributes.isDirectory()) {
            channel.sendResponse(new BytesRestResponse(FORBIDDEN));
            return;
        }
        // We don't serve dir but if index.html exists in dir we should serve it
        file = file.resolve("index.html");
        if (!Files.exists(file) || FileSystemUtils.isHidden(file) || !Files.isRegularFile(file)) {
            channel.sendResponse(new BytesRestResponse(FORBIDDEN));
            return;
        }
    }

    try {
        byte[] data = Files.readAllBytes(file);
        channel.sendResponse(new BytesRestResponse(OK, guessMimeType(sitePath), data));
    } catch (IOException e) {
        channel.sendResponse(new BytesRestResponse(INTERNAL_SERVER_ERROR));
    }
}
 
Example #16
Source File: StaticSite.java    From crate with Apache License 2.0 4 votes vote down vote up
public static FullHttpResponse serveSite(Path siteDirectory,
                                         FullHttpRequest request,
                                         ByteBufAllocator alloc) throws IOException {
    if (request.method() != HttpMethod.GET) {
        return new DefaultFullHttpResponse(HttpVersion.HTTP_1_1, HttpResponseStatus.FORBIDDEN);
    }
    String sitePath = request.uri();
    while (sitePath.length() > 0 && sitePath.charAt(0) == '/') {
        sitePath = sitePath.substring(1);
    }

    // we default to index.html, or what the plugin provides (as a unix-style path)
    // this is a relative path under _site configured by the plugin.
    if (sitePath.length() == 0) {
        sitePath = "index.html";
    }

    final String separator = siteDirectory.getFileSystem().getSeparator();
    // Convert file separators.
    sitePath = sitePath.replace("/", separator);
    Path file = siteDirectory.resolve(sitePath);

    // return not found instead of forbidden to prevent malicious requests to find out if files exist or don't exist
    if (!Files.exists(file) || FileSystemUtils.isHidden(file) ||
        !file.toAbsolutePath().normalize().startsWith(siteDirectory.toAbsolutePath().normalize())) {

        return Responses.contentResponse(
            HttpResponseStatus.NOT_FOUND, alloc, "Requested file [" + file + "] was not found");
    }

    BasicFileAttributes attributes = readAttributes(file, BasicFileAttributes.class);
    if (!attributes.isRegularFile()) {
        // If it's not a regular file, we send a 403
        final String msg = "Requested file [" + file + "] is not a valid file.";
        return Responses.contentResponse(HttpResponseStatus.NOT_FOUND, alloc, msg);
    }
    try {
        byte[] data = Files.readAllBytes(file);
        var resp = Responses.contentResponse(HttpResponseStatus.OK, alloc, data);
        resp.headers().set(HttpHeaderNames.CONTENT_TYPE, guessMimeType(file.toAbsolutePath().toString()));
        return resp;
    } catch (IOException e) {
        return Responses.contentResponse(HttpResponseStatus.INTERNAL_SERVER_ERROR, alloc, e.getMessage());
    }
}
 
Example #17
Source File: FileWatcher.java    From Elasticsearch with Apache License 2.0 4 votes vote down vote up
private Path[] listFiles() throws IOException {
    final Path[] files = FileSystemUtils.files(file);
    Arrays.sort(files);
    return files;
}