Example usage for org.apache.lucene.index IndexWriter close

Introduction

In this page you can find the example usage for org.apache.lucene.index IndexWriter close.

Prototype

@Override
public void close() throws IOException 

Source Link

Document

Closes all open resources and releases the write lock.

Usage

From source file:aos.lucene.tools.BerkeleyDbIndexer.java

License:Apache License

public static void main(String[] args) throws IOException, DatabaseException {
    if (args.length != 1) {
        System.err.println("Usage: BerkeleyDbIndexer <index dir>");
        System.exit(-1);//from  w  ww. jav  a2  s . c o  m
    }

    File indexFile = new File(args[0]);

    if (indexFile.exists()) {
        File[] files = indexFile.listFiles();

        for (int i = 0; i < files.length; i++)
            if (files[i].getName().startsWith("__"))
                files[i].delete();
        indexFile.delete();
    }

    indexFile.mkdir();

    EnvironmentConfig envConfig = new EnvironmentConfig();
    DatabaseConfig dbConfig = new DatabaseConfig();

    envConfig.setTransactional(true);
    envConfig.setInitializeCache(true);
    envConfig.setInitializeLocking(true);
    envConfig.setInitializeLogging(true);
    envConfig.setAllowCreate(true);
    envConfig.setThreaded(true);
    dbConfig.setAllowCreate(true);
    dbConfig.setType(DatabaseType.BTREE);

    Environment env = new Environment(indexFile, envConfig);

    Transaction txn = env.beginTransaction(null, null);
    Database index = env.openDatabase(txn, "__index__", null, dbConfig);
    Database blocks = env.openDatabase(txn, "__blocks__", null, dbConfig);
    txn.commit();

    txn = env.beginTransaction(null, null);
    DbDirectory directory = new DbDirectory(txn, index, blocks);

    IndexWriter writer = new IndexWriter(directory, new StandardAnalyzer(Version.LUCENE_46), true,
            IndexWriter.MaxFieldLength.UNLIMITED);

    Document doc = new Document();
    doc.add(new Field("contents", "The quick brown fox...", Field.Store.YES, Field.Index.ANALYZED));
    writer.addDocument(doc);

    writer.merge(writer.getNextMerge());
    writer.close();

    directory.close();
    txn.commit();

    index.close();
    blocks.close();
    env.close();

    LOGGER.info("Indexing Complete");
}

---

From source file:aos.lucene.tools.BerkeleyDbJEIndexer.java

License:Apache License

public static void main(String[] args) throws IOException, DatabaseException {
    if (args.length != 1) {
        System.err.println("Usage: BerkeleyDbIndexer <index dir>");
        System.exit(-1);// w  w  w  .j  a  va  2 s.  c  o  m
    }

    File indexFile = new File(args[0]);

    if (indexFile.exists()) {
        File[] files = indexFile.listFiles();
        for (int i = 0; i < files.length; i++)

            if (files[i].getName().startsWith("__"))
                files[i].delete();
        indexFile.delete();
    }

    indexFile.mkdir();

    EnvironmentConfig envConfig = new EnvironmentConfig();
    DatabaseConfig dbConfig = new DatabaseConfig();

    envConfig.setTransactional(true);
    envConfig.setAllowCreate(true);
    dbConfig.setTransactional(true);
    dbConfig.setAllowCreate(true);

    Environment env = new Environment(indexFile, envConfig);

    Transaction txn = env.beginTransaction(null, null);
    Database index = env.openDatabase(txn, "__index__", dbConfig);
    Database blocks = env.openDatabase(txn, "__blocks__", dbConfig);
    txn.commit();
    txn = env.beginTransaction(null, null);

    JEDirectory directory = new JEDirectory(txn, index, blocks);

    IndexWriter writer = new IndexWriter(directory, new StandardAnalyzer(Version.LUCENE_46), true,
            IndexWriter.MaxFieldLength.UNLIMITED);

    Document doc = new Document();
    doc.add(new Field("contents", "The quick brown fox...", Field.Store.YES, Field.Index.ANALYZED));
    writer.addDocument(doc);

    writer.merge(writer.getNextMerge());
    writer.close();

    directory.close();
    txn.commit();

    index.close();
    blocks.close();
    env.close();

    LOGGER.info("Indexing Complete");
}

---

From source file:aos.lucene.tools.ChainedFilterTest.java

License:Apache License

@Override
public void setUp() throws Exception {

    directory = new RAMDirectory();

    IndexWriter writer = new IndexWriter(directory, new WhitespaceAnalyzer(Version.LUCENE_46),
            IndexWriter.MaxFieldLength.UNLIMITED);

    Calendar cal = Calendar.getInstance();
    cal.set(2009, 1, 1, 0, 0);/*ww  w.  j  a v  a 2  s.  c om*/

    for (int i = 0; i < MAX; i++) {
        Document doc = new Document();
        doc.add(new Field("key", "" + (i + 1), Field.Store.YES, Field.Index.NOT_ANALYZED));
        doc.add(new Field("owner", (i < MAX / 2) ? "bob" : "sue", Field.Store.YES, Field.Index.NOT_ANALYZED));
        doc.add(new Field("date", DateTools.timeToString(cal.getTimeInMillis(), DateTools.Resolution.DAY),
                Field.Store.YES, Field.Index.NOT_ANALYZED));
        writer.addDocument(doc);

        cal.add(Calendar.DATE, 1);
    }

    writer.close();

    searcher = new IndexSearcher(directory);

    BooleanQuery bq = new BooleanQuery();
    bq.add(new TermQuery(new Term("owner", "bob")), BooleanClause.Occur.SHOULD);
    bq.add(new TermQuery(new Term("owner", "sue")), BooleanClause.Occur.SHOULD);
    query = bq;

    cal.set(2099, 1, 1, 0, 0);
    dateFilter = TermRangeFilter.Less("date",
            DateTools.timeToString(cal.getTimeInMillis(), DateTools.Resolution.DAY));// C

    bobFilter = new CachingWrapperFilter(new QueryWrapperFilter(new TermQuery(new Term("owner", "bob"))));

    sueFilter = new CachingWrapperFilter(new QueryWrapperFilter(new TermQuery(new Term("owner", "sue"))));
}

---

From source file:aos.lucene.tools.FastVectorHighlighterSample.java

License:Apache License

static void makeIndex() throws IOException {
    IndexWriter writer = new IndexWriter(dir, analyzer, true, MaxFieldLength.UNLIMITED);
    for (String d : DOCS) {
        Document doc = new Document();
        doc.add(new Field(F, d, Store.YES, Index.ANALYZED, TermVector.WITH_POSITIONS_OFFSETS));
        writer.addDocument(doc);//www  .  j  a v  a  2s  .c  o  m
    }
    writer.close();
}

---

From source file:api.startup.PDFIndexer.java

License:Open Source License

/**
 * Updates the index/*from  w w  w  .jav a  2  s .c  om*/
 *
 * @throws IOException
 */
public void updateIndex() throws IOException {
    try {
        long startTime = System.nanoTime();
        // Get the index directory
        Directory dir = FSDirectory.open(Paths.get(indexDirectory));
        // Get the directory for resources
        String resourcesDir = resourceDirectory + "/" + Constants.CSV_LOCATION;
        // Get PDF Analyzer
        Analyzer pdf_analyzer = new PDFAnalyzer(resourceDirectory + "/" + Constants.STOPWORDS_FILE);
        // Create an index writer config with the analyzer
        IndexWriterConfig iwc = new IndexWriterConfig(pdf_analyzer);
        // Set the open mode to create or append
        iwc.setOpenMode(IndexWriterConfig.OpenMode.CREATE); // Set index to be created
        // Create an index writer
        IndexWriter writer = new IndexWriter(dir, iwc);
        // Index the documents
        indexDocs(writer, resourcesDir);
        long endTime = System.nanoTime();
        LuceneIndexReader.getInstance().initializeIndexReader(writer);
        writer.close();
        log.info("Took: " + (endTime - startTime) / Math.pow(10, 6) + " milliseconds to generate the index.");
    } catch (IOException e) {
        log.error("IO Exception Thrown while updating index " + e.getMessage() + "\n");
    }

}

---

From source file:aplicacion.sistema.indexer.test.IndexFiles.java

License:Apache License

/** Index all text files under a directory. */
public static void main(String[] args) {
    String usage = "java org.apache.lucene.demo.IndexFiles <root_directory>";

    /*Agustin lo comento 
     * if (args.length == 0) {//from  w w w .  j av  a2 s . co  m
      System.err.println("Usage: " + usage);
      System.exit(1);
    }*/

    if (INDEX_DIR.exists()) {
        System.out.println("Cannot save index to '" + INDEX_DIR + "' directory, please delete it first");
        System.exit(1);
    }

    final File docDir = new File("E:/indexer");
    if (!docDir.exists() || !docDir.canRead()) {
        System.out.println("Document directory '" + docDir.getAbsolutePath()
                + "' does not exist or is not readable, please check the path");
        System.exit(1);
    }

    Date start = new Date();
    try {
        IndexWriter writer = new IndexWriter(FSDirectory.open(INDEX_DIR),
                new StandardAnalyzer(Version.LUCENE_CURRENT), true, IndexWriter.MaxFieldLength.LIMITED);
        System.out.println("Indexing to directory '" + INDEX_DIR + "'...");
        indexDocs(writer, docDir);
        System.out.println("Optimizing...");
        writer.optimize();
        writer.close();

        Date end = new Date();
        System.out.println(end.getTime() - start.getTime() + " total milliseconds");

    } catch (IOException e) {
        System.out.println(" caught a " + e.getClass() + "\n with message: " + e.getMessage());
    }
}

---

From source file:Application.mediaIndexer.java

public static void IndexFiles(String index, String docsPath, TextArea results, boolean CreateOrUpdate,
        boolean removeFiles) throws IOException {
    String indexPath = index;// w ww  .ja  v  a  2  s  . com
    boolean create = CreateOrUpdate;
    final Path docDir = Paths.get(docsPath);
    if (!Files.isReadable(docDir))
        results.appendText("Document directory '" + docDir.toAbsolutePath()
                + "' does not exist or is not readable, please check the path" + "\n");
    Date start = new Date();
    try {
        if (removeFiles)
            results.appendText("Deleting '" + docsPath + "' from directory '" + indexPath + "'..." + "\n");
        // else results.appendText("results '" + docsPath + "' to directory
        // '" + indexPath + "'..." + "\n");
        Directory dir = FSDirectory.open(Paths.get(indexPath));
        Analyzer analyzer = new StandardAnalyzer();
        IndexWriterConfig iwc = new IndexWriterConfig(analyzer);
        if (create)
            iwc.setOpenMode(OpenMode.CREATE);
        else
            iwc.setOpenMode(OpenMode.CREATE_OR_APPEND);
        // Optional: for better results performance, if you are results many
        // documents, increase the RAM buffer.
        // But if you do this, increase the max heap size to the JVM (eg add
        // -Xmx512m or -Xmx1g):
        // iwc.setRAMBufferSizeMB(2048.0);
        IndexWriter writer = new IndexWriter(dir, iwc);
        indexDocs(writer, docDir, results, removeFiles);
        writer.close();
        Date end = new Date();
        long diffInSeconds = (end.getTime() - start.getTime()) / 1000;
        results.appendText(diffInSeconds + " total seconds" + "\n");
    } catch (SAXException e) {
        e.printStackTrace();
    } catch (TikaException e) {
        e.printStackTrace();
    }
}

---

From source file:application.ReviewDocumentIndexer.java

License:Open Source License

/**
 * @param args/*from ww w.j av a  2 s .c o  m*/
 */
@SuppressWarnings("deprecation")
public static void main(String[] args) {
    // Parse command line arguments. Exit program is provided arguments are insufficient
    ReviewDocumentIndexer indexer = new ReviewDocumentIndexer(args);
    if (indexer == null)
        return;

    // Open a new index
    IndexWriter index = null;
    try {
        index = new IndexWriter(new SimpleFSDirectory(new File(Paths.luceneIndex)),
                new ReviewTextAnalyzer(indexer), indexer.new_index ? true : false, MaxFieldLength.UNLIMITED);
        if (indexer.pause_every > 2) {
            index.setMaxBufferedDocs(indexer.pause_every);
        }
        index.setMaxMergeDocs(Config.maxMergeDocs);
        index.setMergeFactor(Config.mergeFactor);
    } catch (CorruptIndexException e) {
        AppLogger.error.log(Level.SEVERE,
                "Lucene detected an inconsistency upon opening the index located at " + Paths.luceneIndex);
        throw new RuntimeException("Exiting application", e);
    } catch (LockObtainFailedException e) {
        AppLogger.error.log(Level.SEVERE,
                "Index located at " + Paths.luceneIndex + " is already open by another Lucene process");
        throw new RuntimeException("Exiting application", e);
    } catch (IOException e) {
        AppLogger.error.log(Level.SEVERE, "Could not access location " + Paths.luceneIndex);
        throw new RuntimeException("Exiting application", e);
    }

    // Load a number of reviews from database
    NumberFormat docIdFormat = TokenListsCollector.defaultDocIdFormat();
    try {
        DatabaseReviewCollection reviews = new DatabaseReviewCollection(indexer.pause_every);
        reviews.setLimits(indexer.min_reviewid, indexer.stop_after);
        int indexed_counter = 0;

        while (reviews.hasNextSegment()) {

            System.out.print(Calendar.getInstance().getTime().toGMTString());

            System.out.print(" Loading from DB... ");
            reviews.loadNextSegment();
            Iterator<Review> reviewsIterator = reviews.getIterator();

            System.out.print(" Indexing... ");
            while (reviewsIterator.hasNext()) {
                DatabaseReview dbr = (DatabaseReview) reviewsIterator.next();
                int dbr_id = dbr.getReviewid();
                int dbr_rating = dbr.getRating();

                try {
                    indexer.theReviewId.set(dbr_id);
                    indexer.theStats.setCurrent(dbr_id, dbr_rating);

                    index.addDocument(dbr.getDocumentForIndexing());
                    indexed_counter++;

                    // Also, keep track of the rating and length of this review
                    indexer.theStats.storeCurrent();

                } catch (CorruptIndexException e) {
                    AppLogger.error.log(Level.SEVERE,
                            "Lucene detected an inconsistency upon saving review #"
                                    + Integer.toString(dbr.getReviewid()) + "to the index located at "
                                    + Paths.luceneIndex);
                    return;
                } catch (IOException e) {
                    AppLogger.error.log(Level.WARNING,
                            "Review #" + Integer.toString(dbr.getReviewid()) + " could not be indexed");
                }
            }

            // Backup everything
            System.out.print("Indexed " + indexed_counter + " reviews total. ");
            if (indexer.pause_every > 0) {
                System.out.print("Saving tokenlists... ");
                indexer.theTokenLists.writeNextFile(docIdFormat);

                System.out.print("Saving state... ");
                try {
                    index.commit();
                    indexer.saveState();
                } catch (CorruptIndexException e) {
                    AppLogger.error.log(Level.SEVERE, "Committing index changes failed on review #"
                            + indexer.theReviewId.get() + "due to CorruptIndexException");
                    return;
                } catch (IOException e) {
                    AppLogger.error.log(Level.WARNING, "Committing index changes failed on review #"
                            + indexer.theReviewId.get() + "due to IOException");
                }
            }
            System.out.print("DONE\n");

            reviews.reset();
        }
    } catch (SQLException e) {
        AppLogger.error.log(Level.SEVERE,
                "An exception occured while trying to access the database.\n" + e.getMessage());
        return;
    }

    try {
        index.close();
        indexer.backupIndex();
    } catch (CorruptIndexException e) {
        // TODO Auto-generated catch block
        e.printStackTrace();
    } catch (IOException e) {
        // TODO Auto-generated catch block
        e.printStackTrace();
    }
    System.err.println("Indexing successfully completed!");
    return;
}

---

From source file:apps.LuceneIndexer.java

License:Apache License

public static void main(String[] args) {
    Options options = new Options();

    options.addOption("i", null, true, "input file");
    options.addOption("o", null, true, "output directory");
    options.addOption("r", null, true, "optional output TREC-format QREL file");

    options.addOption("bm25_b", null, true, "BM25 parameter: b");
    options.addOption("bm25_k1", null, true, "BM25 parameter: k1");
    options.addOption("bm25fixed", null, false, "use the fixed BM25 similarity");

    Joiner commaJoin = Joiner.on(',');
    Joiner spaceJoin = Joiner.on(' ');

    options.addOption("source_type", null, true,
            "document source type: " + commaJoin.join(SourceFactory.getDocSourceList()));

    // If you increase this value, you may need to modify the following line in *.sh file
    // export MAVEN_OPTS="-Xms8192m -server"
    double ramBufferSizeMB = 1024 * 8; // 8 GB

    CommandLineParser parser = new org.apache.commons.cli.GnuParser();

    IndexWriter indexWriter = null;
    BufferedWriter qrelWriter = null;

    int docNum = 0;

    try {/*from  www .j  ava 2 s .co  m*/
        CommandLine cmd = parser.parse(options, args);

        String inputFileName = null, outputDirName = null, qrelFileName = null;

        if (cmd.hasOption("i")) {
            inputFileName = cmd.getOptionValue("i");
        } else {
            Usage("Specify 'input file'", options);
        }

        if (cmd.hasOption("o")) {
            outputDirName = cmd.getOptionValue("o");
        } else {
            Usage("Specify 'index directory'", options);
        }

        if (cmd.hasOption("r")) {
            qrelFileName = cmd.getOptionValue("r");
        }

        String sourceName = cmd.getOptionValue("source_type");

        if (sourceName == null)
            Usage("Specify document source type", options);

        if (qrelFileName != null)
            qrelWriter = new BufferedWriter(new FileWriter(qrelFileName));

        File outputDir = new File(outputDirName);
        if (!outputDir.exists()) {
            if (!outputDir.mkdirs()) {
                System.out.println("couldn't create " + outputDir.getAbsolutePath());
                System.exit(1);
            }
        }
        if (!outputDir.isDirectory()) {
            System.out.println(outputDir.getAbsolutePath() + " is not a directory!");
            System.exit(1);
        }
        if (!outputDir.canWrite()) {
            System.out.println("Can't write to " + outputDir.getAbsolutePath());
            System.exit(1);
        }

        boolean useFixedBM25 = cmd.hasOption("bm25fixed");

        float bm25_k1 = UtilConst.BM25_K1_DEFAULT, bm25_b = UtilConst.BM25_B_DEFAULT;

        if (cmd.hasOption("bm25_k1")) {
            try {
                bm25_k1 = Float.parseFloat(cmd.getOptionValue("bm25_k1"));
            } catch (NumberFormatException e) {
                Usage("Wrong format for 'bm25_k1'", options);
            }
        }

        if (cmd.hasOption("bm25_b")) {
            try {
                bm25_b = Float.parseFloat(cmd.getOptionValue("bm25_b"));
            } catch (NumberFormatException e) {
                Usage("Wrong format for 'bm25_b'", options);
            }
        }

        EnglishAnalyzer analyzer = new EnglishAnalyzer();
        FSDirectory indexDir = FSDirectory.open(Paths.get(outputDirName));
        IndexWriterConfig indexConf = new IndexWriterConfig(analyzer);

        /*
            OpenMode.CREATE creates a new index or overwrites an existing one.
            https://lucene.apache.org/core/6_0_0/core/org/apache/lucene/index/IndexWriterConfig.OpenMode.html#CREATE
        */
        indexConf.setOpenMode(OpenMode.CREATE);
        indexConf.setRAMBufferSizeMB(ramBufferSizeMB);

        System.out.println(String.format("BM25 parameters k1=%f b=%f ", bm25_k1, bm25_b));

        if (useFixedBM25) {
            System.out.println(String.format("Using fixed BM25Simlarity, k1=%f b=%f", bm25_k1, bm25_b));
            indexConf.setSimilarity(new BM25SimilarityFix(bm25_k1, bm25_b));
        } else {
            System.out.println(String.format("Using Lucene BM25Similarity, k1=%f b=%f", bm25_k1, bm25_b));
            indexConf.setSimilarity(new BM25Similarity(bm25_k1, bm25_b));
        }

        indexWriter = new IndexWriter(indexDir, indexConf);

        DocumentSource inpDocSource = SourceFactory.createDocumentSource(sourceName, inputFileName);
        DocumentEntry inpDoc = null;
        TextCleaner textCleaner = new TextCleaner(null);

        while ((inpDoc = inpDocSource.next()) != null) {
            ++docNum;

            Document luceneDoc = new Document();
            ArrayList<String> cleanedToks = textCleaner.cleanUp(inpDoc.mDocText);
            String cleanText = spaceJoin.join(cleanedToks);

            //        System.out.println(inpDoc.mDocId);
            //        System.out.println(cleanText);
            //        System.out.println("==============================");

            luceneDoc.add(new StringField(UtilConst.FIELD_ID, inpDoc.mDocId, Field.Store.YES));
            luceneDoc.add(new TextField(UtilConst.FIELD_TEXT, cleanText, Field.Store.YES));
            indexWriter.addDocument(luceneDoc);

            if (inpDoc.mIsRel != null && qrelWriter != null) {
                saveQrelOneEntry(qrelWriter, inpDoc.mQueryId, inpDoc.mDocId, inpDoc.mIsRel ? MAX_GRADE : 0);
            }
            if (docNum % 1000 == 0)
                System.out.println(String.format("Indexed %d documents", docNum));

        }

    } catch (ParseException e) {
        e.printStackTrace();
        Usage("Cannot parse arguments" + e, options);
    } catch (Exception e) {
        System.err.println("Terminating due to an exception: " + e);
        System.exit(1);
    } finally {
        System.out.println(String.format("Indexed %d documents", docNum));

        try {
            if (null != indexWriter)
                indexWriter.close();
            if (null != qrelWriter)
                qrelWriter.close();
        } catch (IOException e) {
            System.err.println("IO exception: " + e);
            e.printStackTrace();
        }
    }
}

---

From source file:arena.lucene.LuceneDirectoryBean.java

License:Open Source License

public void afterPropertiesSet() throws Exception {
    boolean initialize = false;
    if (this.location == null) {
        this.directory = new RAMDirectory();
        initialize = true;/*from   w ww.ja  v  a2s  .  c  o m*/
    } else {
        File dir = new File(this.servletContext != null ? this.servletContext.getRealPath(location) : location);
        if (!dir.isDirectory()) {
            dir.mkdirs();
        }
        LockFactory lf = (this.disableLocks ? NoLockFactory.getNoLockFactory() : null);
        this.directory = FSDirectory.open(dir, lf);
        log.info("Using lucene directory: file=" + dir + " instance=" + this.directory.toString());
        initialize = (this.initializeIndexIfNew && dir.listFiles().length == 0);
    }

    if (initialize) {
        // create a dummy writer to initialize the index
        IndexWriter writer = null;
        try {
            writer = new IndexWriter(this.directory, new StandardAnalyzer(Version.LUCENE_30), true,
                    MaxFieldLength.LIMITED);
        } finally {
            if (writer != null) {
                writer.close();
            }
        }
    }
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• 37
• 38
• 39
• 40
• 41
• 42
• 43
• 44
• 45
• 46
• 47
• 48
• 49
• »