com.google.common.io.PatternFilenameFilter Java Examples

The following examples show how to use com.google.common.io.PatternFilenameFilter. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: LoadRunner.java    From myrrix-recommender with Apache License 2.0 6 votes vote down vote up
/**
 * @param client recommender to load
 * @param dataDirectory a directory containing data files from which user and item IDs should be read
 * @param steps number of load steps to run
 */
public LoadRunner(MyrrixRecommender client, File dataDirectory, int steps) throws IOException {
  Preconditions.checkNotNull(client);
  Preconditions.checkNotNull(dataDirectory);
  Preconditions.checkArgument(steps > 0);  
  
  log.info("Reading IDs...");    
  FastIDSet userIDsSet = new FastIDSet();
  FastIDSet itemIDsSet = new FastIDSet();
  Splitter comma = Splitter.on(',');
  for (File f : dataDirectory.listFiles(new PatternFilenameFilter(".+\\.csv(\\.(zip|gz))?"))) {
    for (CharSequence line : new FileLineIterable(f)) {
      Iterator<String> it = comma.split(line).iterator();
      userIDsSet.add(Long.parseLong(it.next()));
      itemIDsSet.add(Long.parseLong(it.next()));
    }
  }
  
  this.client = client;    
  this.uniqueUserIDs = userIDsSet.toArray();
  this.uniqueItemIDs = itemIDsSet.toArray();
  this.steps = steps;
}
 
Example #2
Source File: StructSerializationModuleTest.java    From connect-utils with Apache License 2.0 5 votes vote down vote up
@TestFactory
public Stream<DynamicTest> parse() {
  File[] testFiles = testcaseRoot.listFiles(new PatternFilenameFilter("^struct\\d+\\.json$"));

  return Arrays.stream(testFiles)
      .map(f -> dynamicTest(f.getName(), () -> {
        Struct struct = ObjectMapperFactory.INSTANCE.readValue(f, Struct.class);
        assertNotNull(struct);

      }));
}
 
Example #3
Source File: ReconstructionEvaluator.java    From myrrix-recommender with Apache License 2.0 5 votes vote down vote up
private static Multimap<Long,RecommendedItem> readAndCopyDataFiles(File dataDir, File tempDir) throws IOException {
  Multimap<Long,RecommendedItem> data = ArrayListMultimap.create();
  for (File dataFile : dataDir.listFiles(new PatternFilenameFilter(".+\\.csv(\\.(zip|gz))?"))) {
    log.info("Reading {}", dataFile);
    int count = 0;
    for (CharSequence line : new FileLineIterable(dataFile)) {
      Iterator<String> parts = COMMA_TAB_SPLIT.split(line).iterator();
      long userID = Long.parseLong(parts.next());
      long itemID = Long.parseLong(parts.next());
      if (parts.hasNext()) {
        String token = parts.next().trim();
        if (!token.isEmpty()) {
          data.put(userID, new GenericRecommendedItem(itemID, LangUtils.parseFloat(token)));
        }
        // Ignore remove lines
      } else {
        data.put(userID, new GenericRecommendedItem(itemID, 1.0f));
      }
      if (++count % 1000000 == 0) {
        log.info("Finished {} lines", count);
      }
    }

    Files.copy(dataFile, new File(tempDir, dataFile.getName()));
  }
  return data;
}
 
Example #4
Source File: DataPerfectLoader.java    From timbuctoo with GNU General Public License v3.0 5 votes vote down vote up
private File getStructureFile(File dirWithDb) throws InvalidFileException {
  //Look for the STR file
  File[] dbfiles = dirWithDb.listFiles(new PatternFilenameFilter(Pattern.compile(".*\\.STR")));
  if (dbfiles == null || dbfiles.length != 1) {
    throw new InvalidFileException("I expected one .STR file in the zip");
  }
  return dbfiles[0];
}
 
Example #5
Source File: QuidemTest.java    From calcite with Apache License 2.0 5 votes vote down vote up
protected static Collection<Object[]> data(String first) {
  // inUrl = "file:/home/fred/calcite/core/target/test-classes/sql/agg.iq"
  final URL inUrl = JdbcTest.class.getResource("/" + n2u(first));
  final File firstFile = Sources.of(inUrl).file();
  final int commonPrefixLength = firstFile.getAbsolutePath().length() - first.length();
  final File dir = firstFile.getParentFile();
  final List<String> paths = new ArrayList<>();
  final FilenameFilter filter = new PatternFilenameFilter(".*\\.iq$");
  for (File f : Util.first(dir.listFiles(filter), new File[0])) {
    paths.add(f.getAbsolutePath().substring(commonPrefixLength));
  }
  return Lists.transform(paths, path -> new Object[] {path});
}
 
Example #6
Source File: ChromeRecordingWebDriverContainerTest.java    From testcontainers-java with MIT License 5 votes vote down vote up
@Test
public void recordingTestThatShouldBeRecordedAndRetained() {
    File target = vncRecordingDirectory.getRoot();
    try (
        // recordFailing {
        // or if you only want videos for test failures:
        BrowserWebDriverContainer chrome = new BrowserWebDriverContainer()
            .withCapabilities(new ChromeOptions())
            .withRecordingMode(RECORD_FAILING, target)
            // }
            .withRecordingFileFactory(new DefaultRecordingFileFactory())
    ) {
        chrome.start();

        doSimpleExplore(chrome);
        chrome.afterTest(new TestDescription() {
            @Override
            public String getTestId() {
                return getFilesystemFriendlyName();
            }

            @Override
            public String getFilesystemFriendlyName() {
                return "ChromeThatRecordsFailingTests-recordingTestThatShouldBeRecordedAndRetained";
            }
        }, Optional.of(new RuntimeException("Force writing of video file.")));

        String[] files = vncRecordingDirectory.getRoot().list(new PatternFilenameFilter("FAILED-.*\\.flv"));
        assertEquals("Recorded file not found", 1, files.length);
    }

}
 
Example #7
Source File: ChromeRecordingWebDriverContainerTest.java    From testcontainers-java with MIT License 5 votes vote down vote up
@Test
public void recordingTestThatShouldBeRecordedAndRetained() {
    File target = vncRecordingDirectory.getRoot();
    try (
        // recordAll {
        // To do this, simply add extra parameters to the rule constructor:
        BrowserWebDriverContainer chrome = new BrowserWebDriverContainer()
            .withCapabilities(new ChromeOptions())
            .withRecordingMode(RECORD_ALL, target)
            // }
            .withRecordingFileFactory(new DefaultRecordingFileFactory())
    ) {
        chrome.start();

        doSimpleExplore(chrome);
        chrome.afterTest(new TestDescription() {
            @Override
            public String getTestId() {
                return getFilesystemFriendlyName();
            }

            @Override
            public String getFilesystemFriendlyName() {
                return "ChromeThatRecordsAllTests-recordingTestThatShouldBeRecordedAndRetained";
            }
        }, Optional.empty());

        String[] files = vncRecordingDirectory.getRoot().list(new PatternFilenameFilter("PASSED-.*\\.flv"));
        assertEquals("Recorded file not found", 1, files.length);
    }
}
 
Example #8
Source File: AbstractSourceConnectorConfig.java    From kafka-connect-spooldir with Apache License 2.0 5 votes vote down vote up
public AbstractSourceConnectorConfig(ConfigDef definition, Map<?, ?> originals, boolean bufferedInputStream) {
  super(definition, originals);
  this.bufferedInputStream = bufferedInputStream;
  this.inputPath = ConfigUtils.getAbsoluteFile(this, INPUT_PATH_CONFIG);
  this.cleanupPolicy = ConfigUtils.getEnum(CleanupPolicy.class, this, CLEANUP_POLICY_CONF);

  if (finishedPathRequired()) {
    this.finishedPath = ConfigUtils.getAbsoluteFile(this, FINISHED_PATH_CONFIG);
  } else {
    this.finishedPath = null;
  }

  this.errorPath = ConfigUtils.getAbsoluteFile(this, ERROR_PATH_CONFIG);
  this.haltOnError = this.getBoolean(HALT_ON_ERROR_CONF);
  this.minimumFileAgeMS = this.getLong(FILE_MINIMUM_AGE_MS_CONF);
  this.batchSize = this.getInt(BATCH_SIZE_CONF);
  this.topic = this.getString(TOPIC_CONF);
  this.emptyPollWaitMs = this.getLong(EMPTY_POLL_WAIT_MS_CONF);
  this.processingFileExtension = this.getString(PROCESSING_FILE_EXTENSION_CONF);
  this.timestampMode = ConfigUtils.getEnum(TimestampMode.class, this, TIMESTAMP_MODE_CONF);
  final String inputPatternText = this.getString(INPUT_FILE_PATTERN_CONF);
  final Pattern inputPattern = Pattern.compile(inputPatternText);
  this.inputFilenameFilter = new PatternFilenameFilter(inputPattern);
  this.fileSortAttributes = ConfigUtils.getEnums(FileAttribute.class, this, FILE_SORT_ATTRIBUTES_CONF);
  this.taskIndex = getInt(TASK_INDEX_CONF);
  this.taskCount = getInt(TASK_COUNT_CONF);
  this.taskPartitioner = ConfigUtils.getEnum(TaskPartitioner.class, this, TASK_PARTITIONER_CONF);

  if (bufferedInputStream) {
    this.fileBufferSizeBytes = getInt(FILE_BUFFER_SIZE_CONF);
  } else {
    this.fileBufferSizeBytes = 0;
  }
  this.metadataLocation = ConfigUtils.getEnum(MetadataLocation.class, this, METADATA_LOCATION_CONF);
  this.metadataField = this.getString(METADATA_FIELD_CONF);
}
 
Example #9
Source File: QuidemTest.java    From Quicksql with MIT License 5 votes vote down vote up
protected static Collection<Object[]> data(String first) {
  // inUrl = "file:/home/fred/calcite/core/target/test-classes/sql/agg.iq"
  final URL inUrl = JdbcTest.class.getResource("/" + n2u(first));
  final File firstFile = Sources.of(inUrl).file();
  final int commonPrefixLength = firstFile.getAbsolutePath().length() - first.length();
  final File dir = firstFile.getParentFile();
  final List<String> paths = new ArrayList<>();
  final FilenameFilter filter = new PatternFilenameFilter(".*\\.iq$");
  for (File f : Util.first(dir.listFiles(filter), new File[0])) {
    paths.add(f.getAbsolutePath().substring(commonPrefixLength));
  }
  return Lists.transform(paths, path -> new Object[] {path});
}
 
Example #10
Source File: TaggerEmbeddings.java    From easyccg with MIT License 4 votes vote down vote up
public TaggerEmbeddings(File modelFolder, int maxSentenceLength, double beta, int maxTagsPerWord) {
  try {
    FilenameFilter embeddingsFileFilter = new PatternFilenameFilter("embeddings.*");

    // If we're using POS tags or lexical features, load l.
    this.posFeatures = loadSparseFeatures(new File(modelFolder + "/postags"));     
    this.lexicalFeatures = loadSparseFeatures(new File(modelFolder + "/frequentwords"));     
    
    // Load word embeddings.
    embeddingsFeatures = loadEmbeddings(true, modelFolder.listFiles(embeddingsFileFilter));
    
    // Load embeddings for capitalization and suffix features.
    discreteFeatures = new HashMap<String, double[]>();
    discreteFeatures.putAll(loadEmbeddings(false, new File(modelFolder, "capitals")));
    discreteFeatures.putAll(loadEmbeddings(false, new File(modelFolder, "suffix")));
    totalFeatures = 
      (embeddingsFeatures.get(unknownLower).length + 
          discreteFeatures.get(unknownSuffix).length + 
          discreteFeatures.get(capsLower).length +
          posFeatures.size()  + lexicalFeatures.size())       * (2 * contextWindow + 1);
    
    // Load the list of categories used by the model.
    lexicalCategories = loadCategories(new File(modelFolder, "categories"));
    
    // Load the weight matrix used by the classifier.
    weightMatrix = new DenseMatrix(lexicalCategories.size(), totalFeatures);
    loadMatrix(weightMatrix, new File(modelFolder, "classifier"));

    weightMatrixRows = new ArrayList<Vector>(lexicalCategories.size());
    for (int i=0; i<lexicalCategories.size(); i++) {
      Vector row = new DenseVector(totalFeatures);
      for (int j=0; j<totalFeatures; j++) {
        row.set(j, weightMatrix.get(i, j));
      }
      weightMatrixRows.add(row);
    }

    bias = new DenseVector(lexicalCategories.size());
    this.beta = beta;
    this.maxTagsPerWord = maxTagsPerWord;

    int maxCategoryID = 0;
    for (Category c : lexicalCategories) {
      maxCategoryID = Math.max(maxCategoryID, c.getID());
    }
    
    this.tagDict = ImmutableMap.copyOf(loadTagDictionary(modelFolder));
    
    terminalFactory = new SyntaxTreeNodeFactory(maxSentenceLength, maxCategoryID);
    loadVector(bias, new File(modelFolder, "bias"));

  }
  catch (Exception e) {
    throw new RuntimeException(e);
  }
}
 
Example #11
Source File: FileService.java    From website with GNU Affero General Public License v3.0 4 votes vote down vote up
public String[] unprocessedImportItemPdfFiles(ImportItem importItem) {
    File folder = new File(bulkProductDataFolder);
    return folder.list(new PatternFilenameFilter("(?i)" + Pattern.quote(importItem.getIdentifier()) + "\\.(pdf)"));
}
 
Example #12
Source File: FileService.java    From website with GNU Affero General Public License v3.0 4 votes vote down vote up
public String[] unprocessedImportItemHtmlFiles(ImportItem importItem) {
    File folder = new File(bulkProductDataFolder);
    return folder.list(new PatternFilenameFilter("(?i)" + Pattern.quote(importItem.getIdentifier()) + "\\.(htm|html)"));
}
 
Example #13
Source File: FileService.java    From website with GNU Affero General Public License v3.0 4 votes vote down vote up
public String[] existingImportItemFiles(ImportItem importItem) {
    File folder = new File(existingPdfFileFolder);
    return folder.list(new PatternFilenameFilter("(?i)" + Pattern.quote(importItem.getIdentifier()) + "\\.(pdf)"));
}
 
Example #14
Source File: FileService.java    From website with GNU Affero General Public License v3.0 4 votes vote down vote up
public String[] unprocessedImportItemJacketImages(ImportItem importItem) {
    File folder = new File(bulkProductDataFolder);
    logger.debug("unprocessedImportItemJacketImages folder: " + folder.getAbsolutePath());
    return folder.list(new PatternFilenameFilter("(?i)" + Pattern.quote(importItem.getIdentifier()) + "\\.(jpg|png)"));
}
 
Example #15
Source File: ManifestMergerActionTest.java    From bazel with Apache License 2.0 4 votes vote down vote up
@Test public void testMerge() throws Exception {
  String dataDir =
      Paths.get(System.getenv("TEST_WORKSPACE"), System.getenv("TEST_BINARY"))
          .resolveSibling("testing/manifestmerge")
          .toString()
          .replace("\\", "/");

  final Path mergerManifest = rlocation(dataDir + "/merger/AndroidManifest.xml");
  final Path mergeeManifestOne = rlocation(dataDir + "/mergeeOne/AndroidManifest.xml");
  final Path mergeeManifestTwo = rlocation(dataDir + "/mergeeTwo/AndroidManifest.xml");
  assertThat(mergerManifest.toFile().exists()).isTrue();
  assertThat(mergeeManifestOne.toFile().exists()).isTrue();
  assertThat(mergeeManifestTwo.toFile().exists()).isTrue();

  // The following code retrieves the path of the only AndroidManifest.xml in the expected/
  // manifests directory. Unfortunately, this test runs internally and externally and the files
  // have different names.
  final File expectedManifestDirectory =
      mergerManifest.getParent().resolveSibling("expected").toFile();
  final String[] debug =
      expectedManifestDirectory.list(new PatternFilenameFilter(".*AndroidManifest\\.xml$"));
  assertThat(debug).isNotNull();
  final File[] expectedManifestDirectoryManifests =
      expectedManifestDirectory.listFiles((File dir, String name) -> true);
  assertThat(expectedManifestDirectoryManifests).isNotNull();
  assertThat(expectedManifestDirectoryManifests).hasLength(1);
  final Path expectedManifest = expectedManifestDirectoryManifests[0].toPath();

  Files.createDirectories(working.resolve("output"));
  final Path mergedManifest = working.resolve("output/mergedManifest.xml");

  List<String> args =
      generateArgs(
          mergerManifest,
          ImmutableMap.of(mergeeManifestOne, "mergeeOne", mergeeManifestTwo, "mergeeTwo"),
          false, /* isLibrary */
          ImmutableMap.of("applicationId", "com.google.android.apps.testapp"),
          "", /* custom_package */
          mergedManifest);
  ManifestMergerAction.main(args.toArray(new String[0]));

  assertThat(
      Joiner.on(" ")
          .join(Files.readAllLines(mergedManifest, UTF_8))
          .replaceAll("\\s+", " ")
          .trim())
      .isEqualTo(
          Joiner.on(" ")
              .join(Files.readAllLines(expectedManifest, UTF_8))
              .replaceAll("\\s+", " ")
              .trim());
}
 
Example #16
Source File: TaggerEmbeddings.java    From EasySRL with Apache License 2.0 4 votes vote down vote up
public TaggerEmbeddings(final File modelFolder, final double beta, final int maxTagsPerWord,
		final CutoffsDictionaryInterface cutoffs) throws IOException {
	super(cutoffs, beta, loadCategories(new File(modelFolder, "categories")), maxTagsPerWord);
	try {
		final FilenameFilter embeddingsFileFilter = new PatternFilenameFilter("embeddings.*");

		// If we're using POS tags or lexical features, load l.
		this.posFeatures = loadSparseFeatures(new File(modelFolder + "/postags"));
		this.lexicalFeatures = loadSparseFeatures(new File(modelFolder + "/frequentwords"));

		// Load word embeddings.
		embeddingsFeatures = loadEmbeddings(true, modelFolder.listFiles(embeddingsFileFilter));

		// Load embeddings for capitalization and suffix features.
		discreteFeatures = new HashMap<>();
		discreteFeatures.putAll(loadEmbeddings(false, new File(modelFolder, "capitals")));
		discreteFeatures.putAll(loadEmbeddings(false, new File(modelFolder, "suffix")));
		totalFeatures = (embeddingsFeatures.get(unknownLower).length + discreteFeatures.get(unknownSuffix).length
				+ discreteFeatures.get(capsLower).length + posFeatures.size() + lexicalFeatures.size())
				* (2 * contextWindow + 1);

		// Load the list of categories used by the model.
		categoryToIndex = new HashMap<>();
		for (int i = 0; i < lexicalCategories.size(); i++) {
			categoryToIndex.put(lexicalCategories.get(i), i);
		}

		// Load the weight matrix used by the classifier.
		weightMatrix = new DenseMatrix(lexicalCategories.size(), totalFeatures);
		loadMatrix(weightMatrix, new File(modelFolder, "classifier"));

		weightMatrixRows = new ArrayList<>(lexicalCategories.size());
		for (int i = 0; i < lexicalCategories.size(); i++) {
			final Vector row = new DenseVector(totalFeatures);
			for (int j = 0; j < totalFeatures; j++) {
				row.set(j, weightMatrix.get(i, j));
			}
			weightMatrixRows.add(row);
		}

		bias = new DenseVector(lexicalCategories.size());

		loadVector(bias, new File(modelFolder, "bias"));

	} catch (final Exception e) {
		throw new RuntimeException(e);
	}
}
 
Example #17
Source File: AbstractClientTest.java    From myrrix-recommender with Apache License 2.0 4 votes vote down vote up
@Override
@Before
public void setUp() throws Exception {
  super.setUp();

  // Clear some sys properties that may have been set per test
  for (Iterator<?> it = System.getProperties().keySet().iterator(); it.hasNext();) {
    if (it.next().toString().startsWith("model.")) {
      it.remove();
    }
  }

  File tempDir = getTestTempDir();
  File testDataDir = new File(getTestDataPath());
  Preconditions.checkState(testDataDir.exists() && testDataDir.isDirectory(),
                           "%s is not an existing directory", testDataDir.getAbsolutePath());

  log.info("Copying files to {}", tempDir);

  if (savedModelFile == null) {
    log.info("No saved model file, building model");
    File[] srcDataFiles = testDataDir.listFiles(new PatternFilenameFilter("[^.].*"));
    if (srcDataFiles != null) {
      for (File srcDataFile : srcDataFiles) {
        File destFile = new File(tempDir, srcDataFile.getName());
        Files.copy(srcDataFile, destFile);
      }
    }
  } else {
    log.info("Found saved model file {} (size {})", savedModelFile, savedModelFile.length());
    Files.copy(savedModelFile, new File(tempDir, "model.bin.gz"));
  }

  log.info("Configuring recommender...");

  RunnerConfiguration runnerConfig = new RunnerConfiguration();
  runnerConfig.setInstanceID("test");
  runnerConfig.setPort(8090); // Odd ports to avoid conflicts
  if (useSecurity()) {
    runnerConfig.setSecurePort(8453); // Odd ports to avoid conflicts
    runnerConfig.setKeystorePassword("changeit");
    runnerConfig.setKeystoreFile(new File("testdata/keystore"));
    runnerConfig.setUserName("foo");
    runnerConfig.setPassword("bar");
  }
  runnerConfig.setLocalInputDir(tempDir);

  runner = new Runner(runnerConfig);
  runner.call();

  boolean clientSecure = runnerConfig.getKeystoreFile() != null;
  int clientPort = clientSecure ? runnerConfig.getSecurePort() : runnerConfig.getPort();
  MyrrixClientConfiguration clientConfig = new MyrrixClientConfiguration();
  clientConfig.setHost("localhost");
  clientConfig.setPort(clientPort);
  clientConfig.setSecure(clientSecure);
  clientConfig.setKeystorePassword(runnerConfig.getKeystorePassword());
  clientConfig.setKeystoreFile(runnerConfig.getKeystoreFile());
  clientConfig.setUserName(runnerConfig.getUserName());
  clientConfig.setPassword(runnerConfig.getPassword());
  client = new ClientRecommender(clientConfig);

  if (callAwait()) {
    log.info("Waiting for client...");
    client.await();
    log.info("Client ready");
  }
}
 
Example #18
Source File: FilterFilesByExtension.java    From levelup-java-examples with Apache License 2.0 3 votes vote down vote up
@Test
public void find_files_by_extension_guava () {

	Pattern pattern = Pattern.compile("^.*.txt");
	
	FilenameFilter filterByExtension = new PatternFilenameFilter(pattern);
	
	File dir = new File(sourceFileURI);
	File[] files = dir.listFiles(filterByExtension);

	logger.info(Arrays.toString(files));
	
	assertTrue(files.length >= 4);
}