org.apache.lucene.store.FilterDirectory Java Examples

The following examples show how to use org.apache.lucene.store.FilterDirectory. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: DirectoryUtils.java    From Elasticsearch with Apache License 2.0 6 votes vote down vote up
/**
 * Tries to extract the leaf of the {@link Directory} if the directory is a {@link FilterDirectory} and cast
 * it to the given target class or returns the given default value, if the leaf is not assignable to the target class.
 * If the given {@link Directory} is a concrete directory it will treated as a leaf and the above applies.
 */
public static <T extends Directory> T getLeaf(Directory dir, Class<T> targetClass, T defaultValue) {
    Directory d = dir;
    if (dir instanceof FilterDirectory) {
        d = getLeafDirectory((FilterDirectory) dir, targetClass);
    }
    if (d instanceof FileSwitchDirectory) {
        T leaf = getLeaf(((FileSwitchDirectory) d).getPrimaryDir(), targetClass);
        if (leaf == null) {
            d = getLeaf(((FileSwitchDirectory) d).getSecondaryDir(), targetClass, defaultValue);
        } else {
            d = leaf;
        }
    }

    if (d != null && targetClass.isAssignableFrom(d.getClass())) {
        return targetClass.cast(d);
    } else {
        return defaultValue;
    }
}
 
Example #2
Source File: BaseCompoundFormatTestCase.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
public void testPassIOContext() throws IOException {
  final String testfile = "_123.test";
  final IOContext myContext = new IOContext();

  Directory dir = new FilterDirectory(newDirectory()) {
    @Override
    public IndexOutput createOutput(String name, IOContext context) throws IOException {
      assertSame(myContext, context);
      return super.createOutput(name, context);
    }
  };
  SegmentInfo si = newSegmentInfo(dir, "_123");
  try (IndexOutput out = dir.createOutput(testfile, myContext)) {
    CodecUtil.writeIndexHeader(out, "Foo", 0, si.getId(), "suffix");
    out.writeInt(3);
    CodecUtil.writeFooter(out);
  }
  
  si.setFiles(Collections.singleton(testfile));
  si.getCodec().compoundFormat().write(dir, si, myContext);
  dir.close();
}
 
Example #3
Source File: TestUtil.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
public static boolean hasWindowsFS(Directory dir) {
  dir = FilterDirectory.unwrap(dir);
  if (dir instanceof FSDirectory) {
    Path path = ((FSDirectory) dir).getDirectory();
    FileSystem fs = path.getFileSystem();
    while (fs instanceof FilterFileSystem) {
      FilterFileSystem ffs = (FilterFileSystem) fs;
      if (ffs.getParent() instanceof WindowsFS) {
        return true;
      }
      fs = ffs.getDelegate();
    }
  }

  return false;
}
 
Example #4
Source File: TestUtil.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
/** Returns true if VirusCheckingFS is in use and was in fact already enabled */
public static boolean disableVirusChecker(Directory in) {
  Directory dir = FilterDirectory.unwrap(in);
  if (dir instanceof FSDirectory) {

    FileSystem fs = ((FSDirectory) dir).getDirectory().getFileSystem();
    while (fs instanceof FilterFileSystem) {
      FilterFileSystem ffs = (FilterFileSystem) fs;
      if (ffs.getParent() instanceof VirusCheckingFS) {
        VirusCheckingFS vfs = (VirusCheckingFS) ffs.getParent();
        boolean isEnabled = vfs.isEnabled();
        vfs.disable();
        return isEnabled;
      }
      fs = ffs.getDelegate();
    }
  }

  return false;
}
 
Example #5
Source File: TestUtil.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
public static void enableVirusChecker(Directory in) {
  Directory dir = FilterDirectory.unwrap(in);
  if (dir instanceof FSDirectory) {

    FileSystem fs = ((FSDirectory) dir).getDirectory().getFileSystem();
    while (fs instanceof FilterFileSystem) {
      FilterFileSystem ffs = (FilterFileSystem) fs;
      if (ffs.getParent() instanceof VirusCheckingFS) {
        VirusCheckingFS vfs = (VirusCheckingFS) ffs.getParent();
        vfs.enable();
        return;
      }
      fs = ffs.getDelegate();
    }
  }
}
 
Example #6
Source File: TestOfflineSorter.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
/** Make sure corruption on the incoming (unsorted) file is caught, even if the corruption didn't confuse OfflineSorter! */
public void testBitFlippedOnInput1() throws Exception {

  try (Directory dir0 = newMockDirectory()) {

    Directory dir = new FilterDirectory(dir0) {
      @Override
      public IndexOutput createTempOutput(String prefix, String suffix, IOContext context) throws IOException {
        IndexOutput out = in.createTempOutput(prefix, suffix, context);
        if (prefix.equals("unsorted")) {
          return new CorruptingIndexOutput(dir0, 22, out);
        } else {
          return out;
        }
      }
    };

    IndexOutput unsorted = dir.createTempOutput("unsorted", "tmp", IOContext.DEFAULT);
    writeAll(unsorted, generateFixed(10*1024));

    CorruptIndexException e = expectThrows(CorruptIndexException.class, () -> {
        new OfflineSorter(dir, "foo").sort(unsorted.getName());
      });
    assertTrue(e.getMessage().contains("checksum failed (hardware problem?)"));
  }
}
 
Example #7
Source File: TestIndexWriter.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
public void testPendingDeletionsRollbackWithReader() throws IOException {
  // irony: currently we don't emulate windows well enough to work on windows!
  assumeFalse("windows is not supported", Constants.WINDOWS);

  Path path = createTempDir();

  // Use WindowsFS to prevent open files from being deleted:
  FileSystem fs = new WindowsFS(path.getFileSystem()).getFileSystem(URI.create("file:///"));
  Path root = new FilterPath(path, fs);
  try (FSDirectory _dir = new NIOFSDirectory(root)) {
    Directory dir = new FilterDirectory(_dir) {};

    IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
    IndexWriter w = new IndexWriter(dir, iwc);
    Document d = new Document();
    d.add(new StringField("id", "1", Field.Store.YES));
    d.add(new NumericDocValuesField("numval", 1));
    w.addDocument(d);
    w.commit();
    w.addDocument(d);
    w.flush();
    DirectoryReader reader = DirectoryReader.open(w);
    w.rollback();

    // try-delete superfluous files (some will fail due to windows-fs)
    IndexWriterConfig iwc2 = new IndexWriterConfig(new MockAnalyzer(random()));
    new IndexWriter(dir, iwc2).close();

    // test that we can index on top of pending deletions
    IndexWriterConfig iwc3 = new IndexWriterConfig(new MockAnalyzer(random()));
    w = new IndexWriter(dir, iwc3);
    w.addDocument(d);
    w.commit();

    reader.close();
    w.close();
  }
}
 
Example #8
Source File: InternalEngine.java    From crate with Apache License 2.0 5 votes vote down vote up
static Map<String, String> getReaderAttributes(Directory directory) {
    Directory unwrap = FilterDirectory.unwrap(directory);
    boolean defaultOffHeap = FsDirectoryService.isHybridFs(unwrap) || unwrap instanceof MMapDirectory;
    return Map.of(
        // if we are using MMAP for term dics we force all off heap
        BlockTreeTermsReader.FST_MODE_KEY,
        defaultOffHeap ? FSTLoadMode.OFF_HEAP.name() : FSTLoadMode.ON_HEAP.name()
    );
}
 
Example #9
Source File: DirectoryFactory.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
protected Directory getBaseDir(Directory dir) {
  Directory baseDir = dir;
  while (baseDir instanceof FilterDirectory) {
    baseDir = ((FilterDirectory)baseDir).getDelegate();
  } 
  
  return baseDir;
}
 
Example #10
Source File: TestOfflineSorter.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
/** Make sure corruption on a temp file (partition) is caught, even if the corruption didn't confuse OfflineSorter! */
public void testBitFlippedOnPartition1() throws Exception {

  try (Directory dir0 = newMockDirectory()) {

    Directory dir = new FilterDirectory(dir0) {

      boolean corrupted;

      @Override
      public IndexOutput createTempOutput(String prefix, String suffix, IOContext context) throws IOException {
        IndexOutput out = in.createTempOutput(prefix, suffix, context);
        if (corrupted == false && suffix.equals("sort")) {
          corrupted = true;
          return new CorruptingIndexOutput(dir0, 544677, out);
        } else {
          return out;
        }
      }
    };

    IndexOutput unsorted = dir.createTempOutput("unsorted", "tmp", IOContext.DEFAULT);
    writeAll(unsorted, generateFixed((int) (OfflineSorter.MB * 3)));

    CorruptIndexException e = expectThrows(CorruptIndexException.class, () -> {
        new OfflineSorter(dir, "foo", OfflineSorter.DEFAULT_COMPARATOR, BufferSize.megabytes(1), 10, -1, null, 0).sort(unsorted.getName());
      });
    assertTrue(e.getMessage().contains("checksum failed (hardware problem?)"));
  }
}
 
Example #11
Source File: DirectoryUtils.java    From Elasticsearch with Apache License 2.0 5 votes vote down vote up
static final <T extends Directory> Directory getLeafDirectory(FilterDirectory dir, Class<T> targetClass) {
    Directory current = dir.getDelegate();
    while (true) {
        if ((current instanceof FilterDirectory)) {
            if (targetClass != null && targetClass.isAssignableFrom(current.getClass())) {
                break;
            }
            current = ((FilterDirectory) current).getDelegate();
        } else {
            break;
        }
    }
    return current;
}
 
Example #12
Source File: IOUtils.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
/** If the dir is an {@link FSDirectory} or wraps one via possibly
 *  nested {@link FilterDirectory} or {@link FileSwitchDirectory},
 *  this returns {@link #spins(Path)} for the wrapped directory,
 *  else, true.
 *
 *  @throws IOException if {@code path} does not exist.
 *
 *  @lucene.internal */
public static boolean spins(Directory dir) throws IOException {
  dir = FilterDirectory.unwrap(dir);
  if (dir instanceof FileSwitchDirectory) {
    FileSwitchDirectory fsd = (FileSwitchDirectory) dir;
    // Spinning is contagious:
    return spins(fsd.getPrimaryDir()) || spins(fsd.getSecondaryDir());
  } else if (dir instanceof ByteBuffersDirectory) {
    return false;
  } else if (dir instanceof FSDirectory) {
    return spins(((FSDirectory) dir).getDirectory());
  } else {
    return true;
  }
}
 
Example #13
Source File: TestUtil.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
public static boolean hasVirusChecker(Directory dir) {
  dir = FilterDirectory.unwrap(dir);
  if (dir instanceof FSDirectory) {
    return hasVirusChecker(((FSDirectory) dir).getDirectory());
  } else {
    return false;
  }
}
 
Example #14
Source File: TestOfflineSorter.java    From lucene-solr with Apache License 2.0 4 votes vote down vote up
/** Make sure corruption on the incoming (unsorted) file is caught, if the corruption did confuse OfflineSorter! */
public void testBitFlippedOnInput2() throws Exception {

  try (Directory dir0 = newMockDirectory()) {

    Directory dir = new FilterDirectory(dir0) {
      @Override
      public IndexOutput createTempOutput(String prefix, String suffix, IOContext context) throws IOException {
        IndexOutput out = in.createTempOutput(prefix, suffix, context);
        if (prefix.equals("unsorted")) {
          return new CorruptingIndexOutput(dir0, 22, out) {
            @Override
            protected void corruptFile() throws IOException {
              String newTempName;
              try(IndexOutput tmpOut = dir0.createTempOutput("tmp", "tmp", IOContext.DEFAULT);
                  IndexInput in = dir0.openInput(out.getName(), IOContext.DEFAULT)) {
                newTempName = tmpOut.getName();
                // Replace length at the end with a too-long value:
                short v = in.readShort();
                assertEquals(256, v);
                tmpOut.writeShort(Short.MAX_VALUE);
                tmpOut.copyBytes(in, in.length()-Short.BYTES);
              }

              // Delete original and copy corrupt version back:
              dir0.deleteFile(out.getName());
              dir0.copyFrom(dir0, newTempName, out.getName(), IOContext.DEFAULT);
              dir0.deleteFile(newTempName);
            }
          };
        } else {
          return out;
        }
      }
    };

    IndexOutput unsorted = dir.createTempOutput("unsorted", "tmp", IOContext.DEFAULT);
    writeAll(unsorted, generateFixed(5*1024));

    // This corruption made OfflineSorter fail with its own exception, but we verify and throw a CorruptIndexException
    // instead when checksums don't match.
    CorruptIndexException e = expectThrows(CorruptIndexException.class, () -> {
        new OfflineSorter(dir, "foo").sort(unsorted.getName());
      });
    assertTrue(e.getMessage().contains("checksum failed (hardware problem?)"));
  }
}
 
Example #15
Source File: TestIndexWriterMaxDocs.java    From lucene-solr with Apache License 2.0 4 votes vote down vote up
/** 
 * LUCENE-6299: Test if addindexes(Dir[]) prevents exceeding max docs.
 */
// TODO: can we use the setter to lower the amount of docs to be written here?
@Nightly
public void testAddTooManyIndexesDir() throws Exception {
  // we cheat and add the same one over again... IW wants a write lock on each
  Directory dir = newDirectory(random(), NoLockFactory.INSTANCE);
  Document doc = new Document();
  IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(null));
  for (int i = 0; i < 100000; i++) {
    w.addDocument(doc);
  }
  w.forceMerge(1);
  w.commit();
  w.close();
  
  // wrap this with disk full, so test fails faster and doesn't fill up real disks.
  MockDirectoryWrapper dir2 = newMockDirectory();
  w = new IndexWriter(dir2, new IndexWriterConfig(null));
  w.commit(); // don't confuse checkindex
  dir2.setMaxSizeInBytes(dir2.sizeInBytes() + 65536); // 64KB
  Directory dirs[] = new Directory[1 + (IndexWriter.MAX_DOCS / 100000)];
  for (int i = 0; i < dirs.length; i++) {
    // bypass iw check for duplicate dirs
    dirs[i] = new FilterDirectory(dir) {};
  }

  try {
    w.addIndexes(dirs);
    fail("didn't get expected exception");
  } catch (IllegalArgumentException expected) {
    // pass
  } catch (IOException fakeDiskFull) {
    final Exception e;
    if (fakeDiskFull.getMessage() != null && fakeDiskFull.getMessage().startsWith("fake disk full")) {
      e = new RuntimeException("test failed: IW checks aren't working and we are executing addIndexes");
      e.addSuppressed(fakeDiskFull);
    } else {
      e = fakeDiskFull;
    }
    throw e;
  }
  
  w.close();
  dir.close();
  dir2.close();
}
 
Example #16
Source File: TestOfflineSorter.java    From lucene-solr with Apache License 2.0 4 votes vote down vote up
/** Make sure corruption on a temp file (partition) is caught, if the corruption did confuse OfflineSorter! */
public void testBitFlippedOnPartition2() throws Exception {

  try (Directory dir0 = newMockDirectory()) {

    Directory dir = new FilterDirectory(dir0) {

      boolean corrupted;

      @Override
      public IndexOutput createTempOutput(String prefix, String suffix, IOContext context) throws IOException {
        IndexOutput out = in.createTempOutput(prefix, suffix, context);
        if (corrupted == false && suffix.equals("sort")) {
          corrupted = true;
          return new CorruptingIndexOutput(dir0, 544677, out) {
            @Override
            protected void corruptFile() throws IOException {
              String newTempName;
              try(IndexOutput tmpOut = dir0.createTempOutput("tmp", "tmp", IOContext.DEFAULT);
                  IndexInput in = dir0.openInput(out.getName(), IOContext.DEFAULT)) {
                newTempName = tmpOut.getName();
                tmpOut.copyBytes(in, 1025905);
                short v = in.readShort();
                assertEquals(254, v);
                tmpOut.writeShort(Short.MAX_VALUE);
                tmpOut.copyBytes(in, in.length()-1025905-Short.BYTES);
              }

              // Delete original and copy corrupt version back:
              dir0.deleteFile(out.getName());
              dir0.copyFrom(dir0, newTempName, out.getName(), IOContext.DEFAULT);
              dir0.deleteFile(newTempName);
            }
          };
        } else {
          return out;
        }
      }
    };

    IndexOutput unsorted = dir.createTempOutput("unsorted", "tmp", IOContext.DEFAULT);
    writeAll(unsorted, generateFixed((int) (OfflineSorter.MB * 3)));

    CorruptIndexException e = expectThrows(CorruptIndexException.class, () -> {
        new OfflineSorter(dir, "foo", OfflineSorter.DEFAULT_COMPARATOR, BufferSize.megabytes(1), 10, -1, null, 0).sort(unsorted.getName());
      });
    assertTrue(e.getMessage().contains("checksum failed (hardware problem?)"));
  }
}
 
Example #17
Source File: TestBKD.java    From lucene-solr with Apache License 2.0 4 votes vote down vote up
/** Make sure corruption on an input sort file is caught, even if BKDWriter doesn't get angry */
public void testBitFlippedOnPartition1() throws Exception {

  // Generate fixed data set:
  int numDocs = atLeast(10000);
  int numBytesPerDim = 4;
  int numDims = 3;

  byte[][][] docValues = new byte[numDocs][][];
  byte counter = 0;

  for(int docID=0;docID<numDocs;docID++) {
    byte[][] values = new byte[numDims][];
    for(int dim=0;dim<numDims;dim++) {
      values[dim] = new byte[numBytesPerDim];
      for(int i=0;i<values[dim].length;i++) {
        values[dim][i] = counter;
        counter++;
      }
    }
    docValues[docID] = values;
  }

  try (Directory dir0 = newMockDirectory()) {

    Directory dir = new FilterDirectory(dir0) {
      boolean corrupted;
      @Override
      public IndexOutput createTempOutput(String prefix, String suffix, IOContext context) throws IOException {
        IndexOutput out = in.createTempOutput(prefix, suffix, context);
        if (corrupted == false && prefix.equals("_0") && suffix.equals("bkd_left0")) {
          corrupted = true;
          return new CorruptingIndexOutput(dir0, 22, out);
        } else {
          return out;
        }
      }
    };

    CorruptIndexException e = expectThrows(CorruptIndexException.class, () -> {
        verify(dir, docValues, null, numDims, numDims, numBytesPerDim, 50, 0.1);
      });
    assertTrue(e.getMessage().contains("checksum failed (hardware problem?)"));
  }
}
 
Example #18
Source File: TestBKD.java    From lucene-solr with Apache License 2.0 4 votes vote down vote up
/** Make sure corruption on a recursed partition is caught, when BKDWriter does get angry */
public void testBitFlippedOnPartition2() throws Exception {

  // Generate fixed data set:
  int numDocs = atLeast(10000);
  int numBytesPerDim = 4;
  int numDims = 3;

  byte[][][] docValues = new byte[numDocs][][];
  byte counter = 0;

  for(int docID=0;docID<numDocs;docID++) {
    byte[][] values = new byte[numDims][];
    for(int dim=0;dim<numDims;dim++) {
      values[dim] = new byte[numBytesPerDim];
      for(int i=0;i<values[dim].length;i++) {
        values[dim][i] = counter;
        counter++;
      }
    }
    docValues[docID] = values;
  }

  try (Directory dir0 = newMockDirectory()) {

    Directory dir = new FilterDirectory(dir0) {
      boolean corrupted;
      @Override
      public IndexOutput createTempOutput(String prefix, String suffix, IOContext context) throws IOException {
        IndexOutput out = in.createTempOutput(prefix, suffix, context);
        //System.out.println("prefix=" + prefix + " suffix=" + suffix);
        if (corrupted == false && suffix.equals("bkd_left0")) {
          //System.out.println("now corrupt byte=" + x + " prefix=" + prefix + " suffix=" + suffix);
          corrupted = true;
          return new CorruptingIndexOutput(dir0, 22072, out);
        } else {
          return out;
        }
      }
    };

    Throwable t = expectThrows(CorruptIndexException.class, () -> {
        verify(dir, docValues, null, numDims, numDims, numBytesPerDim, 50, 0.1);
      });
    assertCorruptionDetected(t);
  }
}
 
Example #19
Source File: FsDirectoryService.java    From crate with Apache License 2.0 4 votes vote down vote up
public static boolean isHybridFs(Directory directory) {
    Directory unwrap = FilterDirectory.unwrap(directory);
    return unwrap instanceof HybridDirectory;
}