Java Code Examples for org.apache.lucene.store.Directory#fileLength()

private void readFile(Directory dir, String name) throws Exception {
  IndexInput input = dir.openInput(name, newIOContext(random()));
  try {
    long size = dir.fileLength(name);
    long bytesLeft = size;
    while (bytesLeft > 0) {
      final int numToRead;
      if (bytesLeft < buffer.length)
        numToRead = (int) bytesLeft;
      else
        numToRead = buffer.length;
      input.readBytes(buffer, 0, numToRead, false);
      bytesLeft -= numToRead;
    }
    // Don't do this in your real backups!  This is just
    // to force a backup to take a somewhat long time, to
    // make sure we are exercising the fact that the
    // IndexWriter should not delete this file even when I
    // take my time reading it.
    Thread.sleep(1);
  } finally {
    input.close();
  }
}
 

/**
 * Make sure two files are byte-byte identical.
 */
private void assertFilesIdentical(Directory dir, String golden, String sorted) throws IOException {
  long numBytes = dir.fileLength(golden);
  assertEquals(numBytes, dir.fileLength(sorted));

  byte[] buf1 = new byte[64 * 1024];
  byte[] buf2 = new byte[64 * 1024];
  try (
       IndexInput in1 = dir.openInput(golden, IOContext.READONCE);
       IndexInput in2 = dir.openInput(sorted, IOContext.READONCE)
       ) {
    long left = numBytes;
    while (left > 0) {
      int chunk = (int) Math.min(buf1.length, left);
      left -= chunk;
      in1.readBytes(buf1, 0, chunk);
      in2.readBytes(buf2, 0, chunk);
      for (int i = 0; i < chunk; i++) {
        assertEquals(buf1[i], buf2[i]);
      }
    }
  }
}
 

/**
 * All the files which are common between master and slave must have same size and same checksum else we assume
 * they are not compatible (stale).
 *
 * @return true if the index stale and we need to download a fresh copy, false otherwise.
 * @throws IOException  if low level io error
 */
private boolean isIndexStale(Directory dir) throws IOException {
  for (Map<String, Object> file : filesToDownload) {
    String filename = (String) file.get(NAME);
    Long length = (Long) file.get(SIZE);
    Long checksum = (Long) file.get(CHECKSUM);
    if (slowFileExists(dir, filename)) {
      if (checksum != null) {
        if (!(compareFile(dir, filename, length, checksum).equal)) {
          // file exists and size or checksum is different, therefore we must download it again
          return true;
        }
      } else {
        if (length != dir.fileLength(filename)) {
          log.warn("File {} did not match. expected length is {} and actual length is {}",
              filename, length, dir.fileLength(filename));
          return true;
        }
      }
    }
  }
  return false;
}
 

private void addRecoveredFileDetails(SegmentInfos si, Store store, RecoveryState.Index index) throws IOException {
    final Directory directory = store.directory();
    for (String name : Lucene.files(si)) {
        long length = directory.fileLength(name);
        index.addFileDetail(name, length, true);
    }
}
 

private static boolean isValid(HdfsDirectory localDir, Directory remoteDir, String name) throws IOException {
  LastModified lastModified = (LastModified) remoteDir;
  long fileModified = lastModified.getFileModified(name);
  long fileLength = remoteDir.fileLength(name);

  if (localDir.fileExists(name)) {
    LOG.info("Cache file exists [{0}]", name);
    if (localDir.fileLength(name) == fileLength) {
      LOG.info("Cache file length matches [{0}]", name);
      String lastModFile = name + LASTMOD;
      if (localDir.fileExists(lastModFile) && localDir.fileLength(lastModFile) == 8) {
        LOG.info("Cache file last mod file exists [{0}]", name);
        IndexInput input = localDir.openInput(lastModFile, IOContext.DEFAULT);
        long lastMod = input.readLong();
        if (lastMod == fileModified) {
          LOG.info("Cache file last mod matches [{0}]", name);
          return true;
        } else {
          LOG.info("Cache file last mod does not match [{0}]", name);
        }
      } else {
        LOG.info("Cache file last mod file does not exist [{0}]", name);
      }
    } else {
      LOG.info("Cache file length does not match [{0}]", name);
    }
  } else {
    LOG.info("Cache file does not exist [{0}]", name);
  }
  return false;
}
 

private long copy(String file, Directory srcDirectory, HdfsDirectory destDirectory) throws IOException {
  long fileLength = srcDirectory.fileLength(file);
  IOContext context = IOContext.DEFAULT;
  IndexOutput os = null;
  IndexInput is = null;
  IOException priorException = null;
  try {
    os = destDirectory.createOutput(file, context);
    is = srcDirectory.openInput(file, context);
    os.copyBytes(is, is.length());
  } catch (IOException ioe) {
    priorException = ioe;
  } finally {
    boolean success = false;
    try {
      IOUtils.closeWhileHandlingException(priorException, os, is);
      success = true;
    } finally {
      if (!success) {
        try {
          destDirectory.deleteFile(file);
        } catch (Throwable t) {
        }
      }
    }
  }
  return fileLength;
}
 

@Test
public void testEOF() throws IOException {
  Directory fsDir = new RAMDirectory();
  String name = "test.eof";
  createFile(name, fsDir, directory);
  long fsLength = fsDir.fileLength(name);
  long hdfsLength = directory.fileLength(name);
  assertEquals(fsLength, hdfsLength);
  testEof(name, fsDir, fsLength);
  testEof(name, directory, hdfsLength);
}
 

@Test
public void testEOF() throws IOException {
  Directory fsDir = FSDirectory.open(new File(file, "normal"));
  String name = "test.eof";
  createFile(name, fsDir, directory);
  long fsLength = fsDir.fileLength(name);
  long hdfsLength = directory.fileLength(name);
  assertEquals(fsLength, hdfsLength);
  testEof(name, fsDir, fsLength);
  testEof(name, directory, hdfsLength);
}
 

@Test
public void testEOF() throws IOException {
  Directory fsDir = new ByteBuffersDirectory();
  String name = "test.eof";
  createFile(name, fsDir, directory);
  long fsLength = fsDir.fileLength(name);
  long hdfsLength = directory.fileLength(name);
  assertEquals(fsLength, hdfsLength);
  testEof(name,fsDir,fsLength);
  testEof(name,directory,hdfsLength);
}
 

@Test
public void testEOF() throws IOException {
  Directory fsDir = FSDirectory.open(new File(file, "normal").toPath());
  String name = "test.eof";
  createFile(name, fsDir, directory);
  long fsLength = fsDir.fileLength(name);
  long hdfsLength = directory.fileLength(name);
  assertEquals(fsLength, hdfsLength);
  testEof(name, fsDir, fsLength);
  testEof(name, directory, hdfsLength);
  fsDir.close();
}
 

private static long estimateSizeInBytes(Directory directory) throws IOException {
    long estimatedSize = 0;
    String[] files = directory.listAll();
    for (String file : files) {
        try {
            estimatedSize += directory.fileLength(file);
        } catch (NoSuchFileException | FileNotFoundException | AccessDeniedException e) {
            // ignore, the file is not there no more; on Windows, if one thread concurrently deletes a file while
            // calling Files.size, you can also sometimes hit AccessDeniedException
        }
    }
    return estimatedSize;
}
 

public OfflinePointReader(Directory tempDir, String tempFileName, int packedBytesLength, long start, long length, byte[] reusableBuffer) throws IOException {
  this.bytesPerDoc = packedBytesLength + Integer.BYTES;
  this.packedValueLength = packedBytesLength;

  if ((start + length) * bytesPerDoc + CodecUtil.footerLength() > tempDir.fileLength(tempFileName)) {
    throw new IllegalArgumentException("requested slice is beyond the length of this file: start=" + start + " length=" + length + " bytesPerDoc=" + bytesPerDoc + " fileLength=" + tempDir.fileLength(tempFileName) + " tempFileName=" + tempFileName);
  }
  if (reusableBuffer == null) {
    throw new IllegalArgumentException("[reusableBuffer] cannot be null");
  }
  if (reusableBuffer.length < bytesPerDoc) {
    throw new IllegalArgumentException("Length of [reusableBuffer] must be bigger than " + bytesPerDoc);
  }

  this.maxPointOnHeap =  reusableBuffer.length / bytesPerDoc;
  // Best-effort checksumming:
  if (start == 0 && length*bytesPerDoc == tempDir.fileLength(tempFileName) - CodecUtil.footerLength()) {
    // If we are going to read the entire file, e.g. because BKDWriter is now
    // partitioning it, we open with checksums:
    in = tempDir.openChecksumInput(tempFileName, IOContext.READONCE);
  } else {
    // Since we are going to seek somewhere in the middle of a possibly huge
    // file, and not read all bytes from there, don't use ChecksumIndexInput here.
    // This is typically fine, because this same file will later be read fully,
    // at another level of the BKDWriter recursion
    in = tempDir.openInput(tempFileName, IOContext.READONCE);
  }

  name = tempFileName;

  long seekFP = start * bytesPerDoc;
  in.seek(seekFP);
  countLeft = length;
  this.onHeapBuffer = reusableBuffer;
  this.pointValue = new OfflinePointValue(onHeapBuffer, packedValueLength);
}
 

static long dirSize(Directory d) throws IOException {
  long size = 0;
  for (String file : d.listAll()) {
    size += d.fileLength(file);
  }
  return size;
}
 

private static boolean fitsIntoMem(@NonNull final Directory dir) {
    try {
        long size = 0;
        for (String path : dir.listAll()) {
            size+=dir.fileLength(path);
        }
        return IndexCacheFactory.getDefault().getRAMController().shouldLoad(size);
    } catch (IOException ioe) {
        return false;
    }
}
 

public void write(OutputStream out) throws IOException {
  createOutputStream(out);

  IndexInput in = null;
  try {
    initWrite();

    Directory dir = core.withSearcher(searcher -> searcher.getIndexReader().directory());
    in = dir.openInput(fileName, IOContext.READONCE);
    // if offset is mentioned move the pointer to that point
    if (offset != -1) in.seek(offset);

    long filelen = dir.fileLength(fileName);
    long maxBytesBeforePause = 0;

    while (true) {
      offset = offset == -1 ? 0 : offset;
      int read = (int) Math.min(buf.length, filelen - offset);
      in.readBytes(buf, 0, read);

      fos.writeInt(read);
      if (useChecksum) {
        checksum.reset();
        checksum.update(buf, 0, read);
        fos.writeLong(checksum.getValue());
      }
      fos.write(buf, 0, read);
      fos.flush();
      log.debug("Wrote {} bytes for file {}", offset + read, fileName); // logOK

      //Pause if necessary
      maxBytesBeforePause += read;
      if (maxBytesBeforePause >= rateLimiter.getMinPauseCheckBytes()) {
        rateLimiter.pause(maxBytesBeforePause);
        maxBytesBeforePause = 0;
      }
      if (read != buf.length) {
        writeNothingAndFlush();
        fos.close(); // we close because DeflaterOutputStream requires a close call, but but the request outputstream is protected
        break;
      }
      offset += read;
      in.seek(offset);
    }
  } catch (IOException e) {
    log.warn("Exception while writing response for params: {}", params, e);
  } finally {
    if (in != null) {
      in.close();
    }
    extendReserveAndReleaseCommitPoint();
  }
}
 

@Test
public void testRateLimitedReplication() throws Exception {

  //clean index
  masterClient.deleteByQuery("*:*");
  slaveClient.deleteByQuery("*:*");
  masterClient.commit();
  slaveClient.commit();

  masterJetty.stop();
  slaveJetty.stop();

  //Start master with the new solrconfig
  master.copyConfigFile(CONF_DIR + "solrconfig-master-throttled.xml", "solrconfig.xml");
  useFactory(null);
  masterJetty = createAndStartJetty(master);
  masterClient.close();
  masterClient = createNewSolrClient(masterJetty.getLocalPort());

  //index docs
  final int totalDocs = TestUtil.nextInt(random(), 17, 53);
  for (int i = 0; i < totalDocs; i++)
    index(masterClient, "id", i, "name", TestUtil.randomSimpleString(random(), 1000 , 5000));

  masterClient.commit();

  //Check Index Size
  String dataDir = master.getDataDir();
  masterClient.close();
  masterJetty.stop();

  Directory dir = FSDirectory.open(Paths.get(dataDir).resolve("index"));
  String[] files = dir.listAll();
  long totalBytes = 0;
  for(String file : files) {
    totalBytes += dir.fileLength(file);
  }

  float approximateTimeInSeconds = Math.round( totalBytes/1024/1024/0.1 ); // maxWriteMBPerSec=0.1 in solrconfig

  //Start again and replicate the data
  useFactory(null);
  masterJetty = createAndStartJetty(master);
  masterClient = createNewSolrClient(masterJetty.getLocalPort());

  //start slave
  slave.setTestPort(masterJetty.getLocalPort());
  slave.copyConfigFile(CONF_DIR + "solrconfig-slave1.xml", "solrconfig.xml");
  slaveJetty = createAndStartJetty(slave);
  slaveClient.close();
  slaveClient = createNewSolrClient(slaveJetty.getLocalPort());

  long startTime = System.nanoTime();

  pullFromMasterToSlave();

  //Add a few more docs in the master. Just to make sure that we are replicating the correct index point
  //These extra docs should not get replicated
  new Thread(new AddExtraDocs(masterClient, totalDocs)).start();

  //Wait and make sure that it actually replicated correctly.
  @SuppressWarnings({"rawtypes"})
  NamedList slaveQueryRsp = rQuery(totalDocs, "*:*", slaveClient);
  SolrDocumentList slaveQueryResult = (SolrDocumentList) slaveQueryRsp.get("response");
  assertEquals(totalDocs, slaveQueryResult.getNumFound());

  long timeTaken = System.nanoTime() - startTime;

  long timeTakenInSeconds = TimeUnit.SECONDS.convert(timeTaken, TimeUnit.NANOSECONDS);

  //Let's make sure it took more than approximateTimeInSeconds to make sure that it was throttled
  log.info("approximateTimeInSeconds = {} timeTakenInSeconds = {}"
      , approximateTimeInSeconds, timeTakenInSeconds);
  assertTrue(timeTakenInSeconds - approximateTimeInSeconds > 0);
}
 

private void corruptFile(Directory dir, String victim) throws IOException {
  try (BaseDirectoryWrapper dirCopy = newDirectory()) {
    dirCopy.setCheckIndexOnClose(false);

    long victimLength = dir.fileLength(victim);
    long flipOffset = TestUtil.nextLong(random(), 0, victimLength - 1);

    if (VERBOSE) {
      System.out.println("TEST: now corrupt file " + victim + " by changing byte at offset " + flipOffset + " (length= " + victimLength + ")");
    }

    for(String name : dir.listAll()) {
      if (name.equals(victim) == false) {
        dirCopy.copyFrom(dir, name, name, IOContext.DEFAULT);
      } else {
        try (IndexOutput out = dirCopy.createOutput(name, IOContext.DEFAULT);
            IndexInput in = dir.openInput(name, IOContext.DEFAULT)) {
            out.copyBytes(in, flipOffset);
            out.writeByte((byte) (in.readByte() + TestUtil.nextInt(random(), 0x01, 0xFF)));
            out.copyBytes(in, victimLength - flipOffset - 1);
        }
        try (IndexInput in = dirCopy.openInput(name, IOContext.DEFAULT)) {
          try {
            CodecUtil.checksumEntireFile(in);
            System.out.println("TEST: changing a byte in " + victim + " did not update the checksum)");
            return;
          } catch (CorruptIndexException e) {
            // ok
          }
        }
      }
      dirCopy.sync(Collections.singleton(name));
    }

    // corruption must be detected
    expectThrowsAnyOf(Arrays.asList(CorruptIndexException.class, IndexFormatTooOldException.class, IndexFormatTooNewException.class),
        () -> {
          try (IndexReader reader = DirectoryReader.open(dirCopy)) {
            for (LeafReaderContext context : reader.leaves()) {
              context.reader().checkIntegrity();
            }
          }
        }
    );
  }
}
 

private static RevisionFile newRevisionFile(String file, Directory dir) throws IOException {
  RevisionFile revFile = new RevisionFile(file);
  revFile.size = dir.fileLength(file);
  return revFile;
}
 

public static void main(String[] args) throws Exception{
    File samplesFilesDir = new File("build/classes/ensemble/");
    File indexDir = new File("build/classes/ensemble/search/index");
    File docDir = new File("../../../artifacts/sdk/docs/api");
    File samplesDir = new File("src/ensemble/samples");
    // create index
    ///System.out.println("Indexing to directory '" + indexDir + "'...");
    long start = System.currentTimeMillis();
    Directory dir = FSDirectory.open(indexDir);
    Analyzer analyzer = new StandardAnalyzer(Version.LUCENE_31);
    IndexWriterConfig iwc = new IndexWriterConfig(Version.LUCENE_31, analyzer);
    iwc.setOpenMode(OpenMode.CREATE);
    // generate and write index of all java doc and samples
    IndexWriter writer = new IndexWriter(dir, iwc);

    List<String> samplesFileList = new ArrayList<String>();

    indexSamples(writer, samplesDir, samplesFileList);
    try {
        indexJavaDocAllClasses(writer, docDir);
    } catch (Exception e) {
        System.out.println("\nWarning: We were not able to locate the JavaFX API documentation for your build environment.\n"
                + "Ensemble search will not include the API documentation.\n"); 
    }
    writer.close();
    // create a listAll.txt file that is used
    FileWriter listAllOut = new FileWriter(new File(indexDir,"listAll.txt"));
    for (String fileName: dir.listAll()) {
        if (!"listAll.txt".equals(fileName)) { // don't include the "listAll.txt" file
            Long length = dir.fileLength(fileName);
            listAllOut.write(fileName);
            listAllOut.write(':');
            listAllOut.write(length.toString());
            listAllOut.write('\n');
        }
    }
    listAllOut.flush();
    listAllOut.close();

    FileWriter sampleFilesCache = new FileWriter(new File(samplesFilesDir,"samplesAll.txt"));
    for (String oneSample: samplesFileList) {
            sampleFilesCache.write(oneSample);
            sampleFilesCache.write('\n');
    }
    sampleFilesCache.flush();
    sampleFilesCache.close();

    // print time taken
    ///System.out.println(System.currentTimeMillis() - start + " total milliseconds");
}
 

public static void main(String[] args) throws Exception{
    File samplesFilesDir = new File("build/classes/ensemble/");
    File indexDir = new File("build/classes/ensemble/search/index");
    File docDir = new File("../../../artifacts/sdk/docs/api");
    File samplesDir = new File("src/ensemble/samples");
    // create index
    ///System.out.println("Indexing to directory '" + indexDir + "'...");
    long start = System.currentTimeMillis();
    Directory dir = FSDirectory.open(indexDir);
    Analyzer analyzer = new StandardAnalyzer(Version.LUCENE_31);
    IndexWriterConfig iwc = new IndexWriterConfig(Version.LUCENE_31, analyzer);
    iwc.setOpenMode(OpenMode.CREATE);
    // generate and write index of all java doc and samples
    IndexWriter writer = new IndexWriter(dir, iwc);

    List<String> samplesFileList = new ArrayList<String>();

    indexSamples(writer, samplesDir, samplesFileList);
    try {
        indexJavaDocAllClasses(writer, docDir);
    } catch (Exception e) {
        System.out.println("\nWarning: We were not able to locate the JavaFX API documentation for your build environment.\n"
                + "Ensemble search will not include the API documentation.\n"); 
    }
    writer.close();
    // create a listAll.txt file that is used
    FileWriter listAllOut = new FileWriter(new File(indexDir,"listAll.txt"));
    for (String fileName: dir.listAll()) {
        if (!"listAll.txt".equals(fileName)) { // don't include the "listAll.txt" file
            Long length = dir.fileLength(fileName);
            listAllOut.write(fileName);
            listAllOut.write(':');
            listAllOut.write(length.toString());
            listAllOut.write('\n');
        }
    }
    listAllOut.flush();
    listAllOut.close();

    FileWriter sampleFilesCache = new FileWriter(new File(samplesFilesDir,"samplesAll.txt"));
    for (String oneSample: samplesFileList) {
            sampleFilesCache.write(oneSample);
            sampleFilesCache.write('\n');
    }
    sampleFilesCache.flush();
    sampleFilesCache.close();

    // print time taken
    ///System.out.println(System.currentTimeMillis() - start + " total milliseconds");
}