Java Code Examples for org.apache.lucene.index.IndexWriter#deleteDocuments()

@Override
public void execute(Namespace args, PrintStream out) throws Exception {
  int trimPercent = args.getInt("percent");
  
  if (trimPercent < 0 || trimPercent > 100) {
    throw new IllegalArgumentException("invalid percent: " + trimPercent);
  }
  
  IndexWriter writer = ctx.getIndexWriter();    
  if (writer != null) {      
    IndexReader reader = ctx.getIndexReader();
    
    writer.deleteDocuments(buildDeleteQuery(trimPercent, reader.maxDoc()));
    writer.commit();      
    ctx.refreshReader();
    reader = ctx.getIndexReader();
    out.println("trim successful, index now contains: " + reader.numDocs() + " docs.");
  }
  else {
    out.println("unable to open writer, index is in readonly mode");
  }
}
 

@Override
void performAction(IndexSearcherCloseable searcher, IndexWriter writer) throws IOException {
  IterableRow iterableRow = getIterableRow(_rowId, searcher);
  for (UpdateRowAction action : _actions) {
    iterableRow = action.performAction(iterableRow);
  }
  Term term = createRowId(_rowId);
  if (iterableRow != null) {
    RecordToDocumentIterable docsToUpdate = new RecordToDocumentIterable(iterableRow, _fieldManager);
    Iterator<Iterable<Field>> iterator = docsToUpdate.iterator();
    final GenericPeekableIterator<Iterable<Field>> gpi = GenericPeekableIterator.wrap(iterator);
    if (gpi.peek() != null) {
      writer.updateDocuments(term, wrapPrimeDoc(new Iterable<Iterable<Field>>() {
        @Override
        public Iterator<Iterable<Field>> iterator() {
          return gpi;
        }
      }));
    } else {
      writer.deleteDocuments(term);
    }
    _writeRecordsMeter.mark(docsToUpdate.count());
  }
  _writeRowMeter.mark();
}
 

protected void indexRandomShapes(IndexWriter w, Object... shapes) throws Exception {
  Set<Integer> deleted = new HashSet<>();
  for (int id = 0; id < shapes.length; ++id) {
    Document doc = new Document();
    doc.add(newStringField("id", "" + id, Field.Store.NO));
    doc.add(new NumericDocValuesField("id", id));
    if (shapes[id] != null) {
      addShapeToDoc(FIELD_NAME, doc, shapes[id]);
    }
    w.addDocument(doc);
    if (id > 0 && random().nextInt(100) == 42) {
      int idToDelete = random().nextInt(id);
      w.deleteDocuments(new Term("id", ""+idToDelete));
      deleted.add(idToDelete);
      if (VERBOSE) {
        System.out.println("   delete id=" + idToDelete);
      }
    }
  }

  if (randomBoolean()) {
    w.forceMerge(1);
  }
}
 

/**
 * Remove document from the index
 * 
 * @param aSourceDocumentId
 *            The ID of the source document to be removed
 * @param aAnnotationDocumentId
 *            The ID of the annotation document to be removed
 * @param aUser
 *            The owner of the document to be removed
 */
private void deindexDocument(long aSourceDocumentId, long aAnnotationDocumentId, String aUser)
    throws IOException
{
    if (!isCreated()) {
        return;
    }

    log.trace(
            "Removing from index in project [{}]({}). sourceId: {}, annotationId: {}, user: {}",
            project.getName(), project.getId(), aSourceDocumentId, aAnnotationDocumentId,
            aUser);

    IndexWriter indexWriter = getIndexWriter();
    indexWriter.deleteDocuments(new Term(FIELD_ID,
            String.format("%d/%d", aSourceDocumentId, aAnnotationDocumentId)));
    scheduleCommit();
}
 

@Override
public void execute(Namespace args, PrintStream out) throws Exception {
  Query q = null;
  String qstring = args.getString("query");
  try{
    q = luceneContext.getQueryBuilder().build(qstring);
  }
  catch(Exception e){
    out.println("cannot parse query: "+e.getMessage());
    return;
  }
  
  out.println("parsed query: " + q);
  
  if (q != null){
    IndexWriter writer = luceneContext.getIndexWriter();
    if (writer != null) {
      writer.deleteDocuments(q);
      writer.commit();
      luceneContext.refreshReader();
    }
    else {
      out.println("unable to open writer, index is in readonly mode");
    }
  }
}
 

private void applyDeletes(Directory directory, IndexWriter indexWriter, IndexSearcherCloseable searcher,
    String shard, boolean emitDeletes, Configuration configuration) throws IOException {
  DirectoryReader newReader = DirectoryReader.open(directory);
  try {
    List<AtomicReaderContext> newLeaves = newReader.getContext().leaves();
    BlurPartitioner blurPartitioner = new BlurPartitioner();
    Text key = new Text();
    int numberOfShards = _shardContext.getTableContext().getDescriptor().getShardCount();
    int shardId = ShardUtil.getShardIndex(shard);

    Action action = new Action() {
      @Override
      public void found(AtomicReader reader, Bits liveDocs, TermsEnum termsEnum) throws IOException {
        DocsEnum docsEnum = termsEnum.docs(liveDocs, null);
        if (docsEnum.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) {
          indexWriter.deleteDocuments(new Term(BlurConstants.ROW_ID, BytesRef.deepCopyOf(termsEnum.term())));
        }
      }
    };

    LOG.info("Applying deletes for table [{0}] shard [{1}] new reader [{2}]", _table, shard, newReader);
    boolean skipCheckRowIds = isInternal(newReader);
    LOG.info("Skip rowid check [{0}] for table [{1}] shard [{2}] new reader [{3}]", skipCheckRowIds, _table, shard,
        newReader);
    for (AtomicReaderContext context : newLeaves) {
      AtomicReader newAtomicReader = context.reader();
      if (isFastRowIdDeleteSupported(newAtomicReader)) {
        runNewRowIdCheckAndDelete(indexWriter, emitDeletes, blurPartitioner, key, numberOfShards, shardId,
            newAtomicReader, skipCheckRowIds);
      } else {
        runOldMergeSortRowIdCheckAndDelete(emitDeletes, searcher.getIndexReader(), blurPartitioner, key,
            numberOfShards, shardId, action, newAtomicReader);
      }
    }
  } finally {
    newReader.close();
  }
}
 

/**
 * Cancella un documento.
 *
 * @param name Il nome del campo Field da utilizzare per recupero del
 * documento.
 * @param value La chiave mediante il quale è stato indicizzato il
 * documento.
 * @throws ApsSystemException In caso di errore
 */
@Override
public synchronized void delete(String name, String value) throws ApsSystemException {
    try {
        IndexWriter writer = new IndexWriter(this.dir, this.getIndexWriterConfig());
        writer.deleteDocuments(new Term(name, value));
        writer.close();
    } catch (IOException e) {
        logger.error("Error deleting document", e);
        throw new ApsSystemException("Error deleting document", e);
    }
}
 

/**
 * Process single entity update in the FT index.
 *
 * @param iw         index writer
 * @param indexName  index name
 * @param documents  documents to index and PK
 * @param remove     remove only
 * @param indexTime  time of this index (added as field to added documents)
 * @param counts     counts[3] = { added, removed, failed }
 *
 * @throws IOException error
 */
protected void fullTextSearchReindexSingleEntity(final IndexWriter iw,
                                                  final String indexName,
                                                  final Pair<PK, Document[]> documents,
                                                  final boolean remove,
                                                  final long indexTime,
                                                  final long[] counts) throws IOException {

    final PK primaryKey = documents.getFirst();

    // Remove all documents with primary key (could be multiple)
    iw.deleteDocuments(new Term(AdapterUtils.FIELD_PK, String.valueOf(primaryKey)));
    counts[1]++;
    LOGFTQ.trace("Removing {} document _PK:{}", indexName, primaryKey);

    if (!remove) {
        // Add documents
        final FacetsConfig facetsConfig = new FacetsConfig();
        for (final Document document : documents.getSecond()) {
            try {
                LuceneDocumentAdapterUtils.addNumericField(document, AdapterUtils.FIELD_INDEXTIME, indexTime, false);
                for (final IndexableField ixf : document) {
                    if (ixf.fieldType() == SortedSetDocValuesFacetField.TYPE) {
                        SortedSetDocValuesFacetField facetField = (SortedSetDocValuesFacetField) ixf;
                        facetsConfig.setIndexFieldName(facetField.dim, facetField.dim);
                        facetsConfig.setMultiValued(facetField.dim, true); // TODO: revisit this but for now all fields assumed to have multivalue
                    }
                }
                iw.addDocument(facetsConfig.build(document));
                counts[0]++;
            } catch (Exception sde) {
                LOGFTQ.error("Updating {} document _PK:{} failed ... cause: {}", indexName, documents.getFirst(), sde.getMessage());
                counts[2]++;
            }
        }
        LOGFTQ.trace("Updating {} document _PK:{}", indexName, primaryKey);
    }

}
 

public void deleteDocument(Term term) {
    try {
        IndexWriterConfig indexWriterConfig = new IndexWriterConfig(analyzer);
        IndexWriter writter = new IndexWriter(memoryIndex, indexWriterConfig);
        writter.deleteDocuments(term);
        writter.close();
    } catch (IOException e) {
        e.printStackTrace();
    }
}
 

@Test
public void testSuggestOnMostlyDeletedDocuments() throws Exception {
  Analyzer analyzer = new MockAnalyzer(random());
  // using IndexWriter instead of RandomIndexWriter
  IndexWriter iw = new IndexWriter(dir, iwcWithSuggestField(analyzer, "suggest_field"));
  int num = Math.min(1000, atLeast(10));
  for (int i = 1; i <= num; i++) {
    Document document = new Document();
    document.add(new SuggestField("suggest_field", "abc_" + i, i));
    document.add(new StoredField("weight_fld", i));
    document.add(new IntPoint("weight_fld", i));
    iw.addDocument(document);

    if (usually()) {
      iw.commit();
    }
  }

  iw.deleteDocuments(IntPoint.newRangeQuery("weight_fld", 2, Integer.MAX_VALUE));

  DirectoryReader reader = DirectoryReader.open(iw);
  SuggestIndexSearcher indexSearcher = new SuggestIndexSearcher(reader);
  PrefixCompletionQuery query = new PrefixCompletionQuery(analyzer, new Term("suggest_field", "abc_"));
  TopSuggestDocs suggest = indexSearcher.suggest(query, 1, false);
  assertSuggestions(suggest, new Entry("abc_1", 1));

  reader.close();
  iw.close();
}
 

@Test
public void testSuggestOnAllDeletedDocuments() throws Exception {
  Analyzer analyzer = new MockAnalyzer(random());
  // using IndexWriter instead of RandomIndexWriter
  IndexWriter iw = new IndexWriter(dir, iwcWithSuggestField(analyzer, "suggest_field"));
  int num = Math.min(1000, atLeast(10));
  for (int i = 0; i < num; i++) {
    Document document = new Document();
    document.add(new SuggestField("suggest_field", "abc_" + i, i));
    document.add(newStringField("delete", "delete", Field.Store.NO));
    iw.addDocument(document);

    if (usually()) {
      iw.commit();
    }
  }

  iw.deleteDocuments(new Term("delete", "delete"));

  DirectoryReader reader = DirectoryReader.open(iw);
  SuggestIndexSearcher indexSearcher = new SuggestIndexSearcher(reader);
  PrefixCompletionQuery query = new PrefixCompletionQuery(analyzer, new Term("suggest_field", "abc_"));
  TopSuggestDocs suggest = indexSearcher.suggest(query, num, false);
  assertThat(suggest.totalHits.value, equalTo(0L));

  reader.close();
  iw.close();
}
 

@Test
public void testNRTDeletedDocFiltering() throws Exception {
  Analyzer analyzer = new MockAnalyzer(random());
  // using IndexWriter instead of RandomIndexWriter
  IndexWriter iw = new IndexWriter(dir, iwcWithSuggestField(analyzer, "suggest_field"));

  int num = Math.min(1000, atLeast(10));

  int numLive = 0;
  List<Entry> expectedEntries = new ArrayList<>();
  for (int i = 0; i < num; i++) {
    Document document = new Document();
    document.add(new SuggestField("suggest_field", "abc_" + i, num - i));
    if (i % 2 == 0) {
      document.add(newStringField("str_field", "delete", Field.Store.YES));
    } else {
      numLive++;
      expectedEntries.add(new Entry("abc_" + i, num - i));
      document.add(newStringField("str_field", "no_delete", Field.Store.YES));
    }
    iw.addDocument(document);

    if (usually()) {
      iw.commit();
    }
  }

  iw.deleteDocuments(new Term("str_field", "delete"));

  DirectoryReader reader = DirectoryReader.open(iw);
  SuggestIndexSearcher indexSearcher = new SuggestIndexSearcher(reader);
  PrefixCompletionQuery query = new PrefixCompletionQuery(analyzer, new Term("suggest_field", "abc_"));
  TopSuggestDocs suggest = indexSearcher.suggest(query, numLive, false);
  assertSuggestions(suggest, expectedEntries.toArray(new Entry[expectedEntries.size()]));

  reader.close();
  iw.close();
}
 

private void deleteDocument() throws IOException {
  IndexWriter writer = new IndexWriter(dir1, new IndexWriterConfig(new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false)).setOpenMode(OpenMode.APPEND));
  writer.deleteDocuments( new Term( "t_text1", "del" ) );
  // To see negative idf, keep comment the following line
  //writer.forceMerge(1);
  writer.close();
}
 

/**
 * 删除指定博客的索引
 *
 * @param userid
 * @throws Exception
 */
public void deleteIndex(String userid) throws Exception {
  IndexWriter writer = getWriter();
  writer.deleteDocuments(new Term("userid", userid));

  writer.forceMergeDeletes(); // 强制删除
  writer.commit();
  writer.close();
}
 

@Override
public <S, T> void store(Collection<T> toAdd, Collection<S> toDelete, Convertor<? super T, ? extends Document> docConvertor, Convertor<? super S, ? extends Query> queryConvertor, boolean optimize) throws IOException {
    lock.writeLock().lock();
    try {
        final IndexWriter out = getWriter();
        try {
            for (S td : toDelete) {
                out.deleteDocuments(queryConvertor.convert(td));
            }
            if (toAdd.isEmpty()) {
                return;
            }
            for (Iterator<T> it = toAdd.iterator(); it.hasNext();) {
                T entry = it.next();
                it.remove();
                final Document doc = docConvertor.convert(entry);
                out.addDocument(doc);
            }
        } finally {

            try {
                out.close();
            } finally {
                refreshReader();
            }
        }
    } finally {
        lock.writeLock().unlock();
    }
}
 

@Override
public File execute(final File expiredFile) throws IOException {
    // count the number of records and determine the max event id that we are deleting.
    final long numDeleted = 0;
    long maxEventId = -1L;
    try (final RecordReader reader = RecordReaders.newRecordReader(expiredFile, repository.getAllLogFiles(), Integer.MAX_VALUE)) {
        maxEventId = reader.getMaxEventId();
    } catch (final IOException ioe) {
        logger.warn("Failed to obtain max ID present in journal file {}", expiredFile.getAbsolutePath());
    }

    // remove the records from the index
    final List<File> indexDirs = indexConfiguration.getIndexDirectories(expiredFile);
    for (final File indexingDirectory : indexDirs) {
        final Term term = new Term(FieldNames.STORAGE_FILENAME, LuceneUtil.substringBefore(expiredFile.getName(), "."));

        boolean deleteDir = false;
        final EventIndexWriter writer = indexManager.borrowIndexWriter(indexingDirectory);
        try {
            final IndexWriter indexWriter = writer.getIndexWriter();
            indexWriter.deleteDocuments(term);
            indexWriter.commit();
            final int docsLeft = indexWriter.getDocStats().numDocs;
            deleteDir = docsLeft <= 0;
            logger.debug("After expiring {}, there are {} docs left for index {}", expiredFile, docsLeft, indexingDirectory);
        } finally {
            indexManager.returnIndexWriter(writer);
        }

        // we've confirmed that all documents have been removed. Delete the index directory.
        if (deleteDir) {
            indexManager.removeIndex(indexingDirectory);
            indexConfiguration.removeIndexDirectory(indexingDirectory);

            deleteDirectory(indexingDirectory);
            logger.info("Removed empty index directory {}", indexingDirectory);
        }
    }

    // Update the minimum index to 1 more than the max Event ID in this file.
    if (maxEventId > -1L) {
        indexConfiguration.setMinIdIndexed(maxEventId + 1L);
    }

    logger.info("Deleted Indices for Expired Provenance File {} from {} index files; {} documents removed", expiredFile, indexDirs.size(), numDeleted);
    return expiredFile;
}
 

public void testPerFieldCodec() throws Exception {
  
  final int NUM_DOCS = atLeast(173);
  if (VERBOSE) {
    System.out.println("TEST: NUM_DOCS=" + NUM_DOCS);
  }

  BaseDirectoryWrapper dir = newDirectory();
  dir.setCheckIndexOnClose(false); // we use a custom codec provider
  IndexWriter w = new IndexWriter(
      dir,
      newIndexWriterConfig(new MockAnalyzer(random())).
      setCodec(new CustomPerFieldCodec()).
          setMergePolicy(newLogMergePolicy(3))
  );
  Document doc = new Document();
  // uses default codec:
  doc.add(newTextField("field1", "this field uses the standard codec as the test", Field.Store.NO));
  // uses memory codec:
  Field field2 = newTextField("field2", "this field uses the memory codec as the test", Field.Store.NO);
  doc.add(field2);
  
  Field idField = newStringField("id", "", Field.Store.NO);

  doc.add(idField);
  for(int i=0;i<NUM_DOCS;i++) {
    idField.setStringValue(""+i);
    w.addDocument(doc);
    if ((i+1)%10 == 0) {
      w.commit();
    }
  }
  if (VERBOSE) {
    System.out.println("TEST: now delete id=77");
  }
  w.deleteDocuments(new Term("id", "77"));

  IndexReader r = DirectoryReader.open(w);
  
  assertEquals(NUM_DOCS-1, r.numDocs());
  IndexSearcher s = newSearcher(r);
  assertEquals(NUM_DOCS-1, s.count(new TermQuery(new Term("field1", "standard"))));
  assertEquals(NUM_DOCS-1, s.count(new TermQuery(new Term("field2", "memory"))));
  r.close();

  if (VERBOSE) {
    System.out.println("\nTEST: now delete 2nd doc");
  }
  w.deleteDocuments(new Term("id", "44"));

  if (VERBOSE) {
    System.out.println("\nTEST: now force merge");
  }
  w.forceMerge(1);
  if (VERBOSE) {
    System.out.println("\nTEST: now open reader");
  }
  r = DirectoryReader.open(w);
  assertEquals(NUM_DOCS-2, r.maxDoc());
  assertEquals(NUM_DOCS-2, r.numDocs());
  s = newSearcher(r);
  assertEquals(NUM_DOCS-2, s.count(new TermQuery(new Term("field1", "standard"))));
  assertEquals(NUM_DOCS-2, s.count(new TermQuery(new Term("field2", "memory"))));
  assertEquals(1, s.count(new TermQuery(new Term("id", "76"))));
  assertEquals(0, s.count(new TermQuery(new Term("id", "77"))));
  assertEquals(0, s.count(new TermQuery(new Term("id", "44"))));

  if (VERBOSE) {
    System.out.println("\nTEST: now close NRT reader");
  }
  r.close();

  w.close();

  dir.close();
}
 

/**
 * Creates the index.
 *
 * @param configFile the config file
 * @param files the files
 * @throws IOException Signals that an I/O exception has occurred.
 */
private static void createIndex(String configFile,
    HashMap<String, String> files) throws IOException {
  // analyzer
  Map<String, String> paramsCharFilterMtas = new HashMap<>();
  paramsCharFilterMtas.put("type", "file");
  Map<String, String> paramsTokenizer = new HashMap<>();
  paramsTokenizer.put("configFile", configFile);
  Analyzer mtasAnalyzer = CustomAnalyzer
      .builder(Paths.get("docker").toAbsolutePath())
      .addCharFilter("mtas", paramsCharFilterMtas)
      .withTokenizer("mtas", paramsTokenizer).build();
  Map<String, Analyzer> analyzerPerField = new HashMap<>();
  analyzerPerField.put(FIELD_CONTENT, mtasAnalyzer);
  PerFieldAnalyzerWrapper analyzer = new PerFieldAnalyzerWrapper(
      new StandardAnalyzer(), analyzerPerField);
  // indexwriter
  IndexWriterConfig config = new IndexWriterConfig(analyzer);
  config.setUseCompoundFile(false);
  config.setCodec(Codec.forName("MtasCodec"));
  IndexWriter w = new IndexWriter(directory, config);
  // delete
  w.deleteAll();
  // add
  int counter = 0;
  for (Entry<String, String> entry : files.entrySet()) {
    addDoc(w, counter, entry.getKey(), entry.getValue());
    if (counter == 0) {
      w.commit();
    } else {
      addDoc(w, counter, entry.getKey(), entry.getValue());
      addDoc(w, counter, "deletable", entry.getValue());
      w.commit();
      w.deleteDocuments(new Term(FIELD_ID, Integer.toString(counter)));
      w.deleteDocuments(new Term(FIELD_TITLE, "deletable"));
      addDoc(w, counter, entry.getKey(), entry.getValue());
    }
    counter++;
  }
  w.commit();
  // finish
  w.close();
}
 

@Override
public File execute(final File expiredFile) throws IOException {
    // count the number of records and determine the max event id that we are deleting.
    final long numDeleted = 0;
    long maxEventId = -1L;
    try (final RecordReader reader = RecordReaders.newRecordReader(expiredFile, repository.getAllLogFiles(), Integer.MAX_VALUE)) {
        maxEventId = reader.getMaxEventId();
    } catch (final IOException ioe) {
        logger.warn("Failed to obtain max ID present in journal file {}", expiredFile.getAbsolutePath());
    }

    // remove the records from the index
    final List<File> indexDirs = indexConfiguration.getIndexDirectories(expiredFile);
    for (final File indexingDirectory : indexDirs) {
        final Term term = new Term(FieldNames.STORAGE_FILENAME, LuceneUtil.substringBefore(expiredFile.getName(), "."));

        boolean deleteDir = false;
        final EventIndexWriter writer = indexManager.borrowIndexWriter(indexingDirectory);
        try {
            final IndexWriter indexWriter = writer.getIndexWriter();
            indexWriter.deleteDocuments(term);
            indexWriter.commit();
            final int docsLeft = indexWriter.numDocs();
            deleteDir = docsLeft <= 0;
            logger.debug("After expiring {}, there are {} docs left for index {}", expiredFile, docsLeft, indexingDirectory);
        } finally {
            indexManager.returnIndexWriter(writer);
        }

        // we've confirmed that all documents have been removed. Delete the index directory.
        if (deleteDir) {
            indexManager.removeIndex(indexingDirectory);
            indexConfiguration.removeIndexDirectory(indexingDirectory);

            deleteDirectory(indexingDirectory);
            logger.info("Removed empty index directory {}", indexingDirectory);
        }
    }

    // Update the minimum index to 1 more than the max Event ID in this file.
    if (maxEventId > -1L) {
        indexConfiguration.setMinIdIndexed(maxEventId + 1L);
    }

    logger.info("Deleted Indices for Expired Provenance File {} from {} index files; {} documents removed", expiredFile, indexDirs.size(), numDeleted);
    return expiredFile;
}
 

public static void delete(Integer contentId, IndexWriter writer)
		throws CorruptIndexException, IOException, ParseException {
	writer.deleteDocuments(new Term(ID, contentId.toString()));
}