Java Code Examples for org.apache.commons.io.FileUtils#touch()

The following examples show how to use org.apache.commons.io.FileUtils#touch() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: ProjectExplorerRefreshTest.java    From tracecompass with Eclipse Public License 2.0 6 votes vote down vote up
/**
 * Test Refresh after modifying trace content while trace is closed.
 *
 * @throws IOException if an exception occurs
 */
@Test
public void test16_03RefreshClosedTraceContentModified() throws IOException {
    SWTBotTreeItem tracesFolder = SWTBotUtils.selectTracesFolder(fBot, TRACE_PROJECT_NAME);
    SWTBotTreeItem kernelTrace = SWTBotUtils.getTraceProjectItem(fBot, tracesFolder, TEST_FILE_KERNEL.getName());
    SWTBotTreeItem ustTrace = SWTBotUtils.getTraceProjectItem(fBot, tracesFolder, TEST_FILE_UST.getName());
    kernelTrace.contextMenu().menu("Select Trace Type...", "Common Trace Format", "Linux Kernel Trace").click();
    kernelTrace.contextMenu().menu("Open").click();
    SWTBotUtils.activateEditor(fBot, TEST_FILE_KERNEL.getName()).close();
    tracesFolder.contextMenu().menu("Open As Experiment...", "Generic Experiment").click();
    SWTBotUtils.activateEditor(fBot, "Experiment").close();
    SWTBotTreeItem project = SWTBotUtils.selectProject(fBot, TRACE_PROJECT_NAME);
    SWTBotTreeItem experiment = SWTBotUtils.getTraceProjectItem(fBot, project, "Experiments", "Experiment");
    FileUtils.touch(FileUtils.getFile(fTracesFolder, TEST_FILE_KERNEL.getName(), "channel1"));
    FileUtils.deleteQuietly(FileUtils.getFile(fTracesFolder, TEST_FILE_UST.getName(), "channel0"));
    assertTrue(kernelTrace.contextMenu().menuItems().contains("Delete Supplementary Files..."));
    assertTrue(ustTrace.contextMenu().menuItems().contains("Delete Supplementary Files..."));
    assertTrue(experiment.contextMenu().menuItems().contains("Delete Supplementary Files..."));
    refresh(() -> tracesFolder.contextMenu().menu("Refresh").click());
    SWTBotUtils.waitUntil(treeItem -> !treeItem.contextMenu().menuItems().contains("Delete Supplementary Files..."),
            kernelTrace, "Supplementary Files did not get deleted");
    SWTBotUtils.waitUntil(treeItem -> !treeItem.contextMenu().menuItems().contains("Delete Supplementary Files..."),
            ustTrace, "Supplementary Files did not get deleted");
    SWTBotUtils.waitUntil(treeItem -> !treeItem.contextMenu().menuItems().contains("Delete Supplementary Files..."),
            experiment, "Supplementary Files did not get deleted");
}
 
Example 2
Source File: WatchDirRecursiveOnStartTest.java    From n2o-framework with Apache License 2.0 6 votes vote down vote up
/**
 * В папке /dir/sub создаём:
 *  /dir/sub/test.txt
 * Пишем в файл
 *  /dir/sub/test.txt
 * Ожидаем события:
 *  create : /dir/sub/test.txt
 *  modify : /dir/sub
 */
@Test
@Ignore
public void testChangeFilesInSubDir() throws IOException {
    //изменение файла в подпапке
    //папка создаётся до старта вочдира, поэтому изменения в ней должны ловиться
    assertTrue(new File(SUB2_DIR).mkdir());
    watchDir.start();
    FileUtils.touch(new File(SUB2_FILE1));
    FileUtils.write(new File(SUB2_FILE1), "test");

    verify(listener, timeout(200).atLeast(1)).fileCreated(eq(Paths.get(SUB2_FILE1)));
    verify(listener, never()).fileModified(eq(Paths.get(SUB2_DIR)));
    verify(listener, never()).fileModified(eq(Paths.get(SUB2_FILE1)));
    verify(listener, never()).fileDeleted(any(Path.class));
    watchDir.stop();
}
 
Example 3
Source File: ConfigDispatcherOSGiTest.java    From smarthome with Eclipse Public License 2.0 6 votes vote down vote up
@Test
public void whenPropertyValuePairsForAGlobalPIDAreInDifferentFilesPropertiesWillNotBeMerged() throws IOException {
    String configDirectory = configBaseDirectory + SEP + "global_pid_different_files_no_merge_conf";
    String servicesDirectory = "global_pid_different_files_no_merge_services";

    initialize(null);

    cd.processConfigFile(new File(getAbsoluteConfigDirectory(configDirectory, servicesDirectory)));

    // Modify this file, so that we are sure it is the last modified file
    File lastModified = new File(configDirectory + SEP + servicesDirectory + SEP + "global.pid.service.c.file.cfg");
    FileUtils.touch(lastModified);
    cd.processConfigFile(lastModified);

    /*
     * Assert that the configuration is updated only with the property=value
     * pairs which are parsed last:
     */
    verifyNotExistingConfigurationProperty("different.files.global.pid", "first.property");
    verifyNotExistingConfigurationProperty("different.files.global.pid", "second.property");
    verifyValueOfConfigurationProperty("different.files.global.pid", "third.property", "third.value");
}
 
Example 4
Source File: WatchDirTest.java    From n2o-framework with Apache License 2.0 6 votes vote down vote up
/**
 * проверка событие изменения
 * проверить что было событие изменения
 *
 */
@Test
@Ignore //todo почему то не срабатывает тест на https://ci.i-novus.ru/view/util/job/watchdir.master.build/lastBuild/net.n2oapp.watchdir$watchdir/testReport/net.n2oapp.watchdir/WatchDirTest/testEventOnChange/
public void testEventOnChange() throws Exception
{
    FileUtils.touch(new File(path.toString()));

    watchDir.start();

    FileUtils.write(new File(path.toString()), "test");
    Thread.sleep(100);
    verify(listener, timeout(100).atLeast(1)).fileModified(eq(path));

    reset(listener);
    FileUtils.write(new File(path.toString()), "test2");
    verify(listener, timeout(100).atLeast(1)).fileModified(eq(path));

    watchDir.stop();
}
 
Example 5
Source File: PipelineTest.java    From wisdom with Apache License 2.0 6 votes vote down vote up
@Test
public void testAdditionUpdateAndDeleteOfFile() throws IOException {
    File txt = new File(SOURCES, "touch.txt");
    txt.createNewFile();
    assertThat(txt.isFile()).isTrue();
    waitPullPeriod();
    assertThat(textWatcher.added).containsExactly("touch.txt");
    assertThat(mdWatcher.added).isEmpty();

    FileUtils.touch(txt);
    waitPullPeriod();
    assertThat(textWatcher.added).containsExactly("touch.txt");
    assertThat(textWatcher.updated).containsExactly("touch.txt");

    FileUtils.deleteQuietly(txt);
    waitPullPeriod();
    assertThat(textWatcher.added).containsExactly("touch.txt");
    assertThat(textWatcher.updated).containsExactly("touch.txt");
    assertThat(textWatcher.deleted).containsExactly("touch.txt");
}
 
Example 6
Source File: Axis2ServerManager.java    From micro-integrator with Apache License 2.0 6 votes vote down vote up
private File copyServiceToFileSystem(String resourceName, String fileName) throws IOException {
    File file = new File(System.getProperty("basedir") + File.separator + "target" + File.separator + fileName);
    if (file.exists()) {
        FileUtils.deleteQuietly(file);
    }
    FileUtils.touch(file);
    OutputStream os = FileUtils.openOutputStream(file);
    InputStream is = new FileInputStream(
            ExtensionUtils.getSystemResourceLocation() + File.separator + "artifacts" + File.separator + "AXIS2"
                    + File.separator + "config" + File.separator + resourceName);
    if (is != null) {
        byte[] data = new byte[1024];
        int len;
        while ((len = is.read(data)) != -1) {
            os.write(data, 0, len);
        }
        os.flush();
        os.close();
        is.close();
    }
    return file;
}
 
Example 7
Source File: FileSystemExtensionTest.java    From james-project with Apache License 2.0 5 votes vote down vote up
@Test
void extensionShouldDeleteWhenTestCreateNewFiles() throws Exception {
    File baseDir = fileSystemExtension.getFileSystem().getBasedir();
    FileUtils.forceMkdir(baseDir);

    File fileInsideBaseDir = new File(baseDir.getPath() + "/fileInsideBaseDir.temp");
    FileUtils.touch(fileInsideBaseDir);

    fileSystemExtension.afterAll(DUMMY_EXTENSION_CONTEXT);

    assertThat(fileSystemExtension.getFileSystem().getBasedir())
        .doesNotExist();
}
 
Example 8
Source File: EmbeddedKafka.java    From modernmt with Apache License 2.0 5 votes vote down vote up
private void start(String netInterface, int port) throws IOException {
    if (!NetworkUtils.isAvailable(port))
        throw new IOException("Port " + port + " is already in use by another process");

    FileUtils.deleteDirectory(this.runtime);
    FileUtils.forceMkdir(this.runtime);
    deleteClusterId();  // we want to delete saved cluster-id because we reset Zookeeper at every startup

    FileUtils.deleteQuietly(this.logFile);
    FileUtils.touch(this.logFile);

    Process zookeeper = null;
    Process kafka;

    boolean success = false;

    try {
        int zookeperPort = NetworkUtils.getAvailablePort();

        zookeeper = this.startZookeeper(zookeperPort);
        kafka = this.startKafka(netInterface, port, zookeperPort);

        success = true;
    } finally {
        if (!success)
            this.kill(zookeeper, 1, TimeUnit.SECONDS);
    }

    this.subprocesses = Arrays.asList(kafka, zookeeper);
}
 
Example 9
Source File: RangeIndexHandler.java    From incubator-pinot with Apache License 2.0 5 votes vote down vote up
private void createRangeIndexForColumn(ColumnMetadata columnMetadata)
    throws IOException {
  String column = columnMetadata.getColumnName();
  File inProgress = new File(_indexDir, column + ".range.inprogress");
  File rangeIndexFile = new File(_indexDir, column + V1Constants.Indexes.BITMAP_RANGE_INDEX_FILE_EXTENSION);

  if (!inProgress.exists()) {
    // Marker file does not exist, which means last run ended normally.

    if (_segmentWriter.hasIndexFor(column, ColumnIndexType.RANGE_INDEX)) {
      // Skip creating range index if already exists.

      LOGGER.info("Found range index for segment: {}, column: {}", _segmentName, column);
      return;
    }

    // Create a marker file.
    FileUtils.touch(inProgress);
  } else {
    // Marker file exists, which means last run gets interrupted.
    // Remove range index if exists.
    // For v1 and v2, it's the actual range index. For v3, it's the temporary range index.
    FileUtils.deleteQuietly(rangeIndexFile);
  }

  // Create new range index for the column.
  LOGGER.info("Creating new range index for segment: {}, column: {}", _segmentName, column);
  handleDictionaryBasedColumn(columnMetadata);

  // For v3, write the generated range index file into the single file and remove it.
  if (_segmentVersion == SegmentVersion.v3) {
    LoaderUtils.writeIndexToV3Format(_segmentWriter, column, rangeIndexFile, ColumnIndexType.RANGE_INDEX);
  }

  // Delete the marker file.
  FileUtils.deleteQuietly(inProgress);

  LOGGER.info("Created range index for segment: {}, column: {}", _segmentName, column);
}
 
Example 10
Source File: WatchDirTest.java    From n2o-framework with Apache License 2.0 5 votes vote down vote up
/**
 * Режим запуска, не срабатывали события а после запуска срабатывают
 *
 */
@Test
@Ignore
public void testStartMonitoring() throws Exception
{
    FileUtils.touch(new File(path.toString()));
    verify(listener, timeout(100).never()).fileCreated(any(Path.class));

    watchDir.start();

    FileUtils.forceDelete(new File(path.toString()));
    verify(listener, timeout(100).atLeast(1)).fileDeleted(eq(path));

    watchDir.stop();
}
 
Example 11
Source File: WatchDirTest.java    From n2o-framework with Apache License 2.0 5 votes vote down vote up
/**
 * проверка событие создания
 * проверить что было событие на создания
 * как side эффект, после события создания следует событие изменения
 *
 */
@Test
@Ignore
public void testEventOnCreate() throws Exception
{
    watchDir.start();

    FileUtils.touch(new File(path.toString()));
    verify(listener, timeout(100).atLeast(1)).fileCreated(eq(path));
    verify(listener, timeout(100).never()).fileModified(eq(path));

    watchDir.stop();
}
 
Example 12
Source File: SchemaCommandTest.java    From parquet-mr with Apache License 2.0 5 votes vote down vote up
@Test
public void testSchemaCommandOverwriteExistentFile() throws IOException {
  File inputFile = parquetFile();
  File outputFile = new File(getTempFolder(), getClass().getSimpleName() + ".avsc");
  FileUtils.touch(outputFile);
  Assert.assertEquals(0, outputFile.length());
  SchemaCommand command = new SchemaCommand(createLogger());
  command.targets = Arrays.asList(inputFile.getAbsolutePath());
  command.outputPath = outputFile.getAbsolutePath();
  command.overwrite = true;
  command.setConf(new Configuration());
  Assert.assertEquals(0, command.run());
  Assert.assertTrue(0 < outputFile.length());
}
 
Example 13
Source File: JavaReaderToXUnitTest.java    From tutorials with MIT License 5 votes vote down vote up
@Test
public void givenUsingCommonsIO_whenWritingReaderContentsToFile_thenCorrect() throws IOException {
    final Reader initialReader = new CharSequenceReader("CharSequenceReader extends Reader");

    final File targetFile = new File("src/test/resources/targetFile.txt");
    FileUtils.touch(targetFile);
    final byte[] buffer = IOUtils.toByteArray(initialReader);
    FileUtils.writeByteArrayToFile(targetFile, buffer);
    initialReader.close();
}
 
Example 14
Source File: GitAPITestCase.java    From git-client-plugin with MIT License 5 votes vote down vote up
public void test_checkoutBranchFailure() throws Exception {
    w = clone(localMirror());
    File lock = new File(w.repo, ".git/index.lock");
    try {
        FileUtils.touch(lock);
        w.git.checkoutBranch("somebranch", "master");
        fail();
    } catch (GitLockFailedException e) {
        // expected
    } finally {
        lock.delete();
    }
}
 
Example 15
Source File: ReleaseRunnerFileTools.java    From owltools with BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
/**
 * @param base directory
 * @param useReleasesFolder 
 * @param ignoreLock
 * @param handlers
 * @throws IOException
 */
ReleaseRunnerFileTools(File base, boolean useReleasesFolder, boolean ignoreLock, List<LogHandler> handlers) throws IOException {
	super();
	this.handlers = handlers;
	this.useReleasesFolder = useReleasesFolder;
	
	// base
	this.base = base.getCanonicalFile();
	
	
	checkFolder(this.base);
	
	// staging directory
	staging = new File(this.base, STAGING_DIRECTORY_NAME).getCanonicalFile();
	checkFolder(staging);
	
	// lock file
	lockFile = new File(staging, STAGING_DIRECTORY_LOCK_FILE_NAME).getCanonicalFile();
	boolean success = lockFile.createNewFile();
	if (!success && !ignoreLock) {
		if (!forceLock(lockFile)) {
			throw new IOException("Could not lock staging directory via lock file: "+lockFile.getAbsolutePath());
		}
		FileUtils.touch(lockFile);
	}
	logInfo("Using staging folder for release manager: "+staging.getAbsolutePath());
	
	// clean staging
	cleanDirectory(staging, STAGING_DIRECTORY_LOCK_FILE_NAME);
	
	// sub directories
	File subsets = new File(staging, SUBSETS_DIRECTORY_NAME).getCanonicalFile();
	checkFolder(subsets);

	File extensions = new File(staging, EXTENSIONS_DIRECTORY_NAME).getCanonicalFile();
	checkFolder(extensions);
}
 
Example 16
Source File: ElasticDownloader.java    From embedded-elasticsearch with Apache License 2.0 4 votes vote down vote up
private void proceedWithDownload(URL source, File target, File statusFile, int connectionTimeout, int readTimeout) throws IOException {
    logger.info("Downloading {} to {} ...", source, target);
    copyURLToFile(source, target, connectionTimeout, readTimeout, installationDescription.getDownloadProxy());
    FileUtils.touch(statusFile);
    logger.info("Download complete");
}
 
Example 17
Source File: TypeScriptCompilerMojoTest.java    From wisdom with Apache License 2.0 4 votes vote down vote up
@Test
public void testWatching() throws MojoFailureException, MojoExecutionException, IOException, WatchingException, InterruptedException {
    cleanup();

    // Copy animal to animal (do not modify var as it is used by other tests).
    final File originalAnimal = new File(FAKE_PROJECT, "src/main/resources/assets/ts/Animal.ts");
    final File newAnimal = new File(FAKE_PROJECT, "src/main/resources/assets/ts/Animal2.ts");
    final File originalRT = new File(FAKE_PROJECT, "src/main/assets/assets/ts/raytracer.ts");
    String originalAnimalContent = FileUtils.readFileToString(originalAnimal);
    String newContent = originalAnimalContent.replace("Animal", "Animal2")
            .replace("Snake", "Snake2")
            .replace("Horse", "Horse2")
            .replace("tom", "tom2")
            .replace("sam", "sam2");
    FileUtils.write(newAnimal, newContent);

    mojo.execute();

    final File anim = new File(FAKE_PROJECT_TARGET, "classes/assets/ts/Animal2.js");
    assertThat(anim).isFile();
    String content = FileUtils.readFileToString(anim);
    assertThat(content).contains("var sam2 = new Snake2(\"Sammy the Python\");");

    final File rt = new File(FAKE_PROJECT_TARGET, "wisdom/assets/ts/raytracer.js");
    assertThat(rt).isFile();
    content = FileUtils.readFileToString(rt);
    assertThat(content).contains("new Plane(new Vector(0.0, 1.0, 0.0), 0.0, Surfaces.checkerboard),");

    // Delete new animal
    newAnimal.delete();
    mojo.fileDeleted(newAnimal);

    assertThat(anim.isFile()).isFalse();
    // Check that the .d.ts and the .js.map are deleted too
    assertThat(new File(FAKE_PROJECT_TARGET, "classes/ts/Animal2.d.ts")).doesNotExist();
    assertThat(new File(FAKE_PROJECT_TARGET, "classes/ts/Animal2.js.map")).doesNotExist();

    // Recreate the file with another name (same content)
    File newFile = new File(FAKE_PROJECT, "src/main/resources/Animal3.ts");
    newContent = originalAnimalContent.replace("Animal", "Animal3")
            .replace("Snake", "Snake3")
            .replace("Horse", "Horse3")
            .replace("tom", "tom3")
            .replace("sam", "sam3");
    FileUtils.write(newFile, newContent);
    mojo.fileCreated(newFile);
    File var3 = new File(FAKE_PROJECT_TARGET, "classes/Animal3.js");
    assertThat(var3).isFile();
    content = FileUtils.readFileToString(var3);
    assertThat(content).contains("var sam3 = new Snake3(\"Sammy the Python\");");

    // Update link
    long originalLastModified = rt.lastModified();
    FileUtils.touch(originalRT);
    mojo.fileUpdated(originalRT);
    // The file should have been updated
    assertThat(rt.lastModified()).isGreaterThanOrEqualTo(originalLastModified);
}
 
Example 18
Source File: BloomFilterHandler.java    From incubator-pinot with Apache License 2.0 4 votes vote down vote up
private void createBloomFilterForColumn(ColumnMetadata columnMetadata)
    throws Exception {
  String columnName = columnMetadata.getColumnName();

  File bloomFilterFileInProgress = new File(_indexDir, columnName + ".bloom.inprogress");
  File bloomFilterFile = new File(_indexDir, columnName + V1Constants.Indexes.BLOOM_FILTER_FILE_EXTENSION);

  if (!bloomFilterFileInProgress.exists()) {
    // Marker file does not exist, which means last run ended normally.
    if (_segmentWriter.hasIndexFor(columnName, ColumnIndexType.BLOOM_FILTER)) {
      // Skip creating bloom filter index if already exists.
      LOGGER.info("Found bloom filter for segment: {}, column: {}", _segmentName, columnName);
      return;
    }
    // Create a marker file.
    FileUtils.touch(bloomFilterFileInProgress);
  } else {
    // Marker file exists, which means last run gets interrupted.

    // Remove bloom filter file.
    FileUtils.deleteQuietly(bloomFilterFile);
  }

  // Create new bloom filter for the column.
  LOGGER.info("Creating new bloom filter for segment: {}, column: {}", _segmentName, columnName);
  try (BloomFilterCreator creator = new BloomFilterCreator(_indexDir, columnName, columnMetadata.getCardinality())) {
    if (columnMetadata.hasDictionary()) {
      // Read dictionary
      try (BaseImmutableDictionary dictionaryReader = getDictionaryReader(columnMetadata, _segmentWriter)) {
        for (int i = 0; i < dictionaryReader.length(); i++) {
          creator.add(dictionaryReader.get(i));
        }
      }
    } else {
      // Read the forward index
      throw new UnsupportedOperationException("Bloom filters not supported for no dictionary columns");
    }
  }

  // For v3, write the generated bloom filter file into the single file and remove it.
  if (_segmentVersion == SegmentVersion.v3) {
    LoaderUtils.writeIndexToV3Format(_segmentWriter, columnName, bloomFilterFile, ColumnIndexType.BLOOM_FILTER);
  }

  // Delete the marker file.
  FileUtils.deleteQuietly(bloomFilterFileInProgress);
  LOGGER.info("Created bloom filter for segment: {}, column: {}", _segmentName, columnName);
}
 
Example 19
Source File: FileSystemDataStoreTest.java    From DataHubSystem with GNU Affero General Public License v3.0 4 votes vote down vote up
@BeforeClass
public void init () throws IOException, 
   DataStoreLocalArchiveNotExistingException, InterruptedException
{
   tmp = Files.createTempDir();
   tmp.mkdirs ();
   HierarchicalDirectoryBuilder db = 
         new HierarchicalDirectoryBuilder (tmp, 100);
   int i=1000;
   while (i-->0) 
   {
      File root = db.getDirectory ();
      FileUtils.touch (new File(root, "toto.txt"));
      FileUtils.touch (new File(root,
         HierarchicalDirectoryBuilder.DHUS_ENTRY_NAME));
      new File(root, "directory").mkdirs ();
   }
   
   //dataStore.processArchiveSync (false);
   
   incoming_root = new File(cfgManager.getArchiveConfiguration ().getIncomingConfiguration ().getPath ());
   
   int maxfileno = cfgManager.getArchiveConfiguration ().getIncomingConfiguration ().getMaxFileNo ();
   
   String[] folders_to_test_true =
   {
      incoming_root.getPath () + "/X1",
      incoming_root.getPath () + "/X2",
      incoming_root.getPath () + "/X3",
      incoming_root.getPath () + "/X4",
      incoming_root.getPath () + "/X1/X1/X1/X1/X1/X1/X1/X1/X1",
      incoming_root.getPath () + "/X1/X2/X3/X4/X5/X6/X7/X8/X9",
      incoming_root.getPath () + "/X1/X1/X1/XAF/X1/X1/X1/X1/X1/",
      incoming_root.getPath () + File.separator + Long.toHexString (maxfileno-1).substring (1),
      incoming_root.getPath () + "/X1/X1/X1/X1/X1/X1/" + HierarchicalDirectoryBuilder.DHUS_ENTRY_NAME,
      incoming_root.getPath () + "/X1/X1/X1/X1/X1/X1/" + HierarchicalDirectoryBuilder.DHUS_ENTRY_NAME + File.separator + IncomingManager.INCOMING_PRODUCT_DIR,
   };
   
   String[] folders_to_test_false =
   {
      "/tmp/toto",
      incoming_root.getPath () + "/my_data",
      incoming_root.getPath () + "/X2/X23453/X2/" + HierarchicalDirectoryBuilder.DHUS_ENTRY_NAME,
      incoming_root.getPath () + "/X3/" + HierarchicalDirectoryBuilder.DHUS_ENTRY_NAME + "/data",
      incoming_root.getPath () + "/X1/X1/X1/X1/X1/X1/" + HierarchicalDirectoryBuilder.DHUS_ENTRY_NAME + File.separator + IncomingManager.INCOMING_PRODUCT_DIR + File.separator + "mydata",
      incoming_root.getPath () + File.separator + maxfileno + "X1/X2/X3",
      incoming_root.getPath () + "/X3/data",
      incoming_root.getPath () + "/A4",
   };

   folders_to_test[0]=folders_to_test_true;
   folders_to_test[1]=folders_to_test_false;
   incomingPathChecker = new IncomingPathChecker ();

}
 
Example 20
Source File: InvertedIndexHandler.java    From incubator-pinot with Apache License 2.0 4 votes vote down vote up
private void createInvertedIndexForColumn(ColumnMetadata columnMetadata)
    throws IOException {
  String column = columnMetadata.getColumnName();

  File inProgress = new File(_indexDir, column + ".inv.inprogress");
  File invertedIndexFile = new File(_indexDir, column + V1Constants.Indexes.BITMAP_INVERTED_INDEX_FILE_EXTENSION);

  if (!inProgress.exists()) {
    // Marker file does not exist, which means last run ended normally.

    if (_segmentWriter.hasIndexFor(column, ColumnIndexType.INVERTED_INDEX)) {
      // Skip creating inverted index if already exists.

      LOGGER.info("Found inverted index for segment: {}, column: {}", _segmentName, column);
      return;
    }

    // Create a marker file.
    FileUtils.touch(inProgress);
  } else {
    // Marker file exists, which means last run gets interrupted.

    // Remove inverted index if exists.
    // For v1 and v2, it's the actual inverted index. For v3, it's the temporary inverted index.
    FileUtils.deleteQuietly(invertedIndexFile);
  }

  // Create new inverted index for the column.
  LOGGER.info("Creating new inverted index for segment: {}, column: {}", _segmentName, column);
  int numDocs = columnMetadata.getTotalDocs();
  try (OffHeapBitmapInvertedIndexCreator creator = new OffHeapBitmapInvertedIndexCreator(_indexDir,
      columnMetadata.getFieldSpec(), columnMetadata.getCardinality(), numDocs,
      columnMetadata.getTotalNumberOfEntries())) {
    try (DataFileReader fwdIndex = getForwardIndexReader(columnMetadata, _segmentWriter)) {
      if (columnMetadata.isSingleValue()) {
        // Single-value column.

        FixedBitSingleValueReader svFwdIndex = (FixedBitSingleValueReader) fwdIndex;
        for (int i = 0; i < numDocs; i++) {
          creator.add(svFwdIndex.getInt(i));
        }
      } else {
        // Multi-value column.

        SingleColumnMultiValueReader mvFwdIndex = (SingleColumnMultiValueReader) fwdIndex;
        int[] dictIds = new int[columnMetadata.getMaxNumberOfMultiValues()];
        for (int i = 0; i < numDocs; i++) {
          int length = mvFwdIndex.getIntArray(i, dictIds);
          creator.add(dictIds, length);
        }
      }
      creator.seal();
    }
  }

  // For v3, write the generated inverted index file into the single file and remove it.
  if (_segmentVersion == SegmentVersion.v3) {
    LoaderUtils.writeIndexToV3Format(_segmentWriter, column, invertedIndexFile, ColumnIndexType.INVERTED_INDEX);
  }

  // Delete the marker file.
  FileUtils.deleteQuietly(inProgress);

  LOGGER.info("Created inverted index for segment: {}, column: {}", _segmentName, column);
}