Example usage for org.apache.lucene.index IndexWriter close

List of usage examples for org.apache.lucene.index IndexWriter close

  1. HOME
  2. Java
  3. org.apache.lucene
  4. org.apache.lucene.index.*
  5. IndexWriter
  6. close

Introduction

In this page you can find the example usage for org.apache.lucene.index IndexWriter close.

Prototype

@Override
public void close() throws IOException

Source Link

Document

Closes all open resources and releases the write lock.

Usage

From source file:com.czw.search.lucene.example.xmlparser.FormBasedXmlQueryDemo.java

License:Apache License

private void openExampleIndex() throws IOException {
    //Create a RAM-based index from our test data file
    RAMDirectory rd = new RAMDirectory();
    IndexWriterConfig iwConfig = new IndexWriterConfig(analyzer);
    IndexWriter writer = new IndexWriter(rd, iwConfig);
    InputStream dataIn = getServletContext().getResourceAsStream("/WEB-INF/data.tsv");
    BufferedReader br = new BufferedReader(new InputStreamReader(dataIn, StandardCharsets.UTF_8));
    String line = br.readLine();//from  w  w w  .  j a va 2s  .c  o  m
    final FieldType textNoNorms = new FieldType(TextField.TYPE_STORED);
    textNoNorms.setOmitNorms(true);
    while (line != null) {
        line = line.trim();
        if (line.length() > 0) {
            //parse row and create a document
            StringTokenizer st = new StringTokenizer(line, "\t");
            Document doc = new Document();
            doc.add(new Field("location", st.nextToken(), textNoNorms));
            doc.add(new Field("salary", st.nextToken(), textNoNorms));
            doc.add(new Field("type", st.nextToken(), textNoNorms));
            doc.add(new Field("description", st.nextToken(), textNoNorms));
            writer.addDocument(doc);
        }
        line = br.readLine();
    }
    writer.close();

    //open searcher
    // this example never closes it reader!
    IndexReader reader = DirectoryReader.open(rd);
    searcher = new IndexSearcher(reader);
}

From source file:com.daniel.IndexFiles.java

License:Apache License

/** Index all text files under a directory. */
public static void main(String[] args) {
    String usage = "java org.apache.lucene.demo.IndexFiles"
            + " [-index INDEX_PATH] [-docs DOCS_PATH] [-update]\n\n"
            + "This indexes the documents in DOCS_PATH, creating a Lucene index"
            + "in INDEX_PATH that can be searched with SearchFiles";
    String indexPath = "index";
    String docsPath = null;//from w ww.  j a va 2s.  c  o m
    boolean create = true;
    for (int i = 0; i < args.length; i++) {
        if ("-index".equals(args[i])) {
            indexPath = args[i + 1];
            i++;
        } else if ("-docs".equals(args[i])) {
            docsPath = args[i + 1];
            i++;
        } else if ("-update".equals(args[i])) {
            create = false;
        }
    }

    if (docsPath == null) {
        System.err.println("Usage: " + usage);
        System.exit(1);
    }

    final File docDir = new File(docsPath);
    if (!docDir.exists() || !docDir.canRead()) {
        System.out.println("Document directory '" + docDir.getAbsolutePath()
                + "' does not exist or is not readable, please check the path");
        System.exit(1);
    }

    Date start = new Date();
    try {
        System.out.println("Indexing to directory '" + indexPath + "'...");

        Directory dir = FSDirectory.open(new File(indexPath));
        Analyzer analyzer = new StandardAnalyzer(Version.LUCENE_40);
        IndexWriterConfig iwc = new IndexWriterConfig(Version.LUCENE_40, analyzer);

        if (create) {
            // Create a new index in the directory, removing any
            // previously indexed documents:
            iwc.setOpenMode(OpenMode.CREATE);
        } else {
            // Add new documents to an existing index:
            iwc.setOpenMode(OpenMode.CREATE_OR_APPEND);
        }

        // Optional: for better indexing performance, if you
        // are indexing many documents, increase the RAM
        // buffer.  But if you do this, increase the max heap
        // size to the JVM (eg add -Xmx512m or -Xmx1g):
        //
        // iwc.setRAMBufferSizeMB(256.0);

        IndexWriter writer = new IndexWriter(dir, iwc);
        indexDocs(writer, docDir);

        // NOTE: if you want to maximize search performance,
        // you can optionally call forceMerge here.  This can be
        // a terribly costly operation, so generally it's only
        // worth it when your index is relatively static (ie
        // you're done adding documents to it):
        //
        // writer.forceMerge(1);

        writer.close();

        Date end = new Date();
        System.out.println(end.getTime() - start.getTime() + " total milliseconds");

    } catch (IOException e) {
        System.out.println(" caught a " + e.getClass() + "\n with message: " + e.getMessage());
    }
}

From source file:com.dasasian.chok.lucene.integration.LuceneClientTest.java

License:Apache License

@Test
public void testFieldSortWithNoResultShard() throws Exception {
    String indexName = "sortIndex";

    File sortIndex = temporaryFolder.newFolder(indexName);
    File sortShard1 = new File(sortIndex, "sortIndex1");
    File sortShard2 = new File(sortIndex, "sortIndex2");
    IndexWriter indexWriter1 = new IndexWriter(FSDirectory.open(sortShard1),
            new StandardAnalyzer(Version.LUCENE_30), true, MaxFieldLength.UNLIMITED);
    IndexWriter indexWriter2 = new IndexWriter(FSDirectory.open(sortShard2),
            new StandardAnalyzer(Version.LUCENE_30), true, MaxFieldLength.UNLIMITED);

    Document document = new Document();
    document.add(new Field("text", "abc", Field.Store.YES, Index.NOT_ANALYZED));
    document.add(new NumericField("timesort", Field.Store.YES, false).setLongValue(1234567890123l));
    indexWriter1.addDocument(document);/*from  w ww  .j a  v a2s  .  c o  m*/
    indexWriter1.close();

    document = new Document();
    document.add(new Field("text", "abc2", Field.Store.YES, Index.NOT_ANALYZED));
    document.add(new NumericField("timesort", Field.Store.YES, false).setLongValue(1234567890123l));
    indexWriter2.addDocument(document);
    indexWriter2.close();

    miniCluster.deployIndex(indexName, sortIndex, 1);

    // query and compare results
    LuceneClient client = new LuceneClient(miniCluster.createInteractionProtocol());
    Sort sort = new Sort(new SortField[] { new SortField("timesort", SortField.LONG) });

    // query both documents
    Query query = new QueryParser(Version.LUCENE_30, "", new KeywordAnalyzer()).parse("text:ab*");
    Hits hits = client.search(query, null, 20, sort);
    assertEquals(2, hits.size());

    // query only one document
    query = new QueryParser(Version.LUCENE_30, "", new KeywordAnalyzer()).parse("text:abc2");
    hits = client.search(query, null, 20, sort);
    assertEquals(1, hits.size());

    // query only one document on one node
    miniCluster.shutdownNode(0);
    TestUtil.waitUntilIndexBalanced(miniCluster.getProtocol(), indexName);
    query = new QueryParser(Version.LUCENE_30, "", new KeywordAnalyzer()).parse("text:abc2");
    hits = client.search(query, null, 20, sort);
    assertEquals(1, hits.size());
    client.close();
}

From source file:com.dasasian.chok.lucene.testutil.LuceneIndexGenerator.java

License:Apache License

public void createIndex(TestIndex testIndex, String[] wordList, int wordsPerDoc, int docsPerShard) {
    long startTime = System.currentTimeMillis();
    try {/*from  www. j  a v  a  2  s. c  o  m*/
        for (File index : testIndex.getShardFiles()) {
            int count = wordList.length;
            Random random = new Random(System.currentTimeMillis());
            IndexWriter indexWriter = new IndexWriter(FSDirectory.open(index),
                    new StandardAnalyzer(Version.LUCENE_30), true, IndexWriter.MaxFieldLength.UNLIMITED);
            for (int i = 0; i < docsPerShard; i++) {
                // generate text first
                StringBuilder text = new StringBuilder();
                for (int j = 0; j < wordsPerDoc; j++) {
                    text.append(wordList[random.nextInt(count)]);
                    text.append(" ");
                }

                Document document = new Document();
                document.add(new Field("key", "key_" + i, Field.Store.NO, Field.Index.NOT_ANALYZED));
                document.add(new Field("text", text.toString(), Field.Store.NO, Field.Index.ANALYZED));
                indexWriter.addDocument(document);

            }
            indexWriter.optimize();
            indexWriter.close();
            System.out.println("Index created with : " + docsPerShard + " documents in "
                    + (System.currentTimeMillis() - startTime) + " ms");

            // when we are ready we move the index to the final destination and write
            // a done flag file we can use in shell scripts to identify the move is
            // done.

            new File(index, "done").createNewFile();
        }

    } catch (Exception e) {
        throw new RuntimeException("Unable to write index", e);
    }
}

From source file:com.devb.search.IndicIndexer.java

License:Apache License

@Override
public void makeIndex() {
    String indexPath = servletContext.getRealPath("/") + "/hindex/";
    String docsPath = servletContext.getRealPath("/") + "/hdocs/";
    boolean create = true;

    final File docDir = new File(docsPath);
    if (!docDir.exists() || !docDir.canRead()) {
        System.out.println("Document directory '" + docDir.getAbsolutePath()
                + "' does not exist or is not readable, please check the path\n");
        return;//from w  w  w. j a v  a  2 s .co  m
    }

    Date start = new Date();
    try {
        System.out.println("Indexing to directory '" + indexPath + "'...\n");

        org.apache.lucene.store.Directory dir = FSDirectory.open(new File(indexPath));
        Analyzer analyzer = new HindiAnalyzer();
        IndexWriterConfig iwc = new IndexWriterConfig(null, analyzer);

        if (create) {
            iwc.setOpenMode(OpenMode.CREATE);
        } else {
            iwc.setOpenMode(OpenMode.CREATE_OR_APPEND);
        }

        IndexWriter writer = new IndexWriter(dir, iwc);
        if (docDir.canRead()) {
            if (docDir.isDirectory()) {
                String[] files = docDir.list();
                if (files != null) {
                    for (int i = 0; i < files.length; i++) {
                        File file = new File(docDir, files[i]);
                        FileInputStream fileInputStream = new FileInputStream(file);
                        BufferedReader reader = new BufferedReader(
                                new InputStreamReader(fileInputStream, "UTF-8"));
                        Tokenizer tokenizer = new StandardTokenizer(reader);
                        CharTermAttribute termAtt = tokenizer.addAttribute(CharTermAttribute.class);
                        tokenizer.reset();
                        int lineNumber = 0;
                        try {
                            while (tokenizer.incrementToken()) {
                                Document doc = new Document();
                                Field pathField = new StringField("path", file.getName(), Field.Store.YES);
                                doc.add(pathField);
                                TextField nField = new TextField("linenumber",
                                        new Integer(++lineNumber).toString(), Store.YES);
                                doc.add(nField);
                                TextField field = new TextField("contents", termAtt.toString(), Store.YES);
                                doc.add(field);
                                writer.addDocument(doc);
                            }
                            System.out.println("Adding " + file + "\n");
                        } catch (Exception e) {
                            e.printStackTrace();
                        } finally {
                            tokenizer.close();
                            reader.close();
                            fileInputStream.close();
                        }
                    }
                }
            }
        }

        writer.close();

        Date end = new Date();
        System.out.println((end.getTime() - start.getTime()) + " total milliseconds\n");

    } catch (IOException e) {
        System.out.println("Caught a " + e.getClass() + "\n with message: " + e.getMessage());
    }
}

From source file:com.devb.search.StandardIndexer.java

License:Apache License

@Override
public void makeIndex() {
    String indexPath = servletContext.getRealPath("/") + "/index/";
    String docsPath = servletContext.getRealPath("/") + "/docs/";
    boolean create = true;

    final File docDir = new File(docsPath);
    if (!docDir.exists() || !docDir.canRead()) {
        System.out.println("Document directory '" + docDir.getAbsolutePath()
                + "' does not exist or is not readable, please check the path\n");
        return;//  ww  w .j  a v  a2 s .co  m
    }

    Date start = new Date();
    try {
        System.out.println("Indexing to directory '" + indexPath + "'...\n");

        org.apache.lucene.store.Directory dir = FSDirectory.open(new File(indexPath));
        Analyzer analyzer = new StandardAnalyzer();
        IndexWriterConfig iwc = new IndexWriterConfig(null, analyzer);

        if (create) {
            iwc.setOpenMode(OpenMode.CREATE);
        } else {
            iwc.setOpenMode(OpenMode.CREATE_OR_APPEND);
        }

        IndexWriter writer = new IndexWriter(dir, iwc);
        if (docDir.canRead()) {
            if (docDir.isDirectory()) {
                String[] files = docDir.list();
                if (files != null) {
                    for (int i = 0; i < files.length; i++) {
                        File file = new File(docDir, files[i]);
                        FileReader fr = new FileReader(file);
                        BufferedReader br = new BufferedReader(fr);
                        String line;
                        int lineNumber = 0;
                        try {
                            while ((line = br.readLine()) != null) {
                                Document doc = new Document();
                                Field pathField = new StringField("path", file.getName(), Field.Store.YES);
                                doc.add(pathField);
                                TextField nField = new TextField("linenumber",
                                        new Integer(++lineNumber).toString(), Store.YES);
                                doc.add(nField);
                                TextField field = new TextField("contents", line, Store.YES);
                                doc.add(field);
                                writer.addDocument(doc);
                            }
                            System.out.println("Adding " + file + "\n");
                        } catch (Exception e) {
                            e.printStackTrace();
                        } finally {
                            br.close();
                            fr.close();
                        }
                    }
                }
            }
        }

        writer.close();

        Date end = new Date();
        System.out.println((end.getTime() - start.getTime()) + " total milliseconds\n");

    } catch (IOException e) {
        System.out.println("Caught a " + e.getClass() + "\n with message: " + e.getMessage());
    }

}

From source file:com.discursive.jccook.xml.bardsearch.CreateIndex.java

License:Apache License

public static void main(String[] pArgs) throws Exception {
    File dataDir = new File("./data/shakespeare");
    logger.info("Looking for XML files in " + dataDir.getAbsolutePath());
    FilenameFilter xmlFilter = new GlobFilenameFilter("*.xml");
    File[] xmlFiles = dataDir.listFiles(xmlFilter);

    logger.info("Creating Index");
    IndexWriter writer = new IndexWriter("index", new StandardAnalyzer(), true);
    PlayIndexer playIndexer = new PlayIndexer(writer);
    playIndexer.init();// w  ww. j  a va 2 s.  c  om
    for (int i = 0; i < xmlFiles.length; i++) {
        System.out.println("Indexing: " + xmlFiles[i]);
        playIndexer.index(xmlFiles[i]);
    }
    writer.optimize();
    writer.close();
    logger.info("Parsing Complete, Index Created");
}

From source file:com.docdoku.server.IndexerBean.java

License:Open Source License

@Asynchronous
@Lock(LockType.WRITE)/*from w  w w  .j  a v a  2s.  c  o  m*/
public void removeFromIndex(String fullName) {
    IndexWriter indexWriter = null;
    Directory indexDir = null;
    try {
        indexDir = FSDirectory.open(new File(indexPath));
        indexWriter = new IndexWriter(indexDir, new StandardAnalyzer(Version.LUCENE_30),
                IndexWriter.MaxFieldLength.LIMITED);
        indexWriter.deleteDocuments(new Term("fullName", fullName));
    } catch (LockObtainFailedException ex) {
        try {
            if (IndexWriter.isLocked(indexDir)) {
                IndexWriter.unlock(indexDir);
            }
        } catch (IOException pIOEx) {
            throw new EJBException(pIOEx);
        }
        throw new EJBException(ex);
    } catch (CorruptIndexException ex) {
        throw new EJBException(ex);
    } catch (IOException ex) {
        throw new EJBException(ex);
    } finally {
        try {
            if (indexWriter != null) {
                indexWriter.close();
            }
        } catch (IOException ex) {
            throw new EJBException(ex);
        }
    }
}

From source file:com.docdoku.server.IndexerBean.java

License:Open Source License

@Asynchronous
@Lock(LockType.WRITE)/*from  w ww  .  ja v  a2s  .c  om*/
public void addToIndex(String fullName, String pathName) {
    IndexWriter indexWriter = null;
    Directory indexDir = null;
    try {
        indexDir = FSDirectory.open(new File(indexPath));
        indexWriter = new IndexWriter(indexDir, new StandardAnalyzer(Version.LUCENE_30),
                IndexWriter.MaxFieldLength.LIMITED);
        int ext = pathName.lastIndexOf('.');
        String extension = "";
        if (ext != -1) {
            extension = pathName.substring(ext);
        }

        if (extension.equals(".odt") || extension.equals(".ods") || extension.equals(".odp")
                || extension.equals(".odg") || extension.equals(".odc") || extension.equals(".odf")
                || extension.equals(".odb") || extension.equals(".odi") || extension.equals(".odm")) {
            final StringBuilder text = new StringBuilder();
            ZipInputStream zipOpenDoc = new ZipInputStream(
                    new BufferedInputStream(new FileInputStream(pathName)));
            ZipEntry zipEntry;
            while ((zipEntry = zipOpenDoc.getNextEntry()) != null) {
                if (zipEntry.getName().equals("content.xml")) {
                    SAXParserFactory saxParserFactory = SAXParserFactory.newInstance();
                    SAXParser parser = saxParserFactory.newSAXParser();
                    parser.parse(zipOpenDoc, new DefaultHandler() {

                        @Override
                        public void characters(char[] ch, int start, int length) throws SAXException {
                            for (int i = start; i < start + length; i++) {
                                text.append(ch[i]);
                            }
                            text.append("\r\n");
                        }
                    });
                    break;
                }
            }
            zipOpenDoc.close();
            Reader contentReader = new StringReader(text.toString());
            addDoc(indexWriter, contentReader, fullName);
            contentReader.close();
        } else if (extension.equals(".doc")) {
            //MSWord Document
            InputStream wordStream = new BufferedInputStream(new FileInputStream(pathName));
            WordExtractor wordExtractor = new WordExtractor(wordStream);
            Reader contentReader = new StringReader(wordExtractor.getText());
            wordStream.close();
            addDoc(indexWriter, contentReader, fullName);
            contentReader.close();
        } else if (extension.equals(".ppt") || extension.equals(".pps")) {
            //MSPowerPoint Document
            InputStream pptStream = new BufferedInputStream(new FileInputStream(pathName));
            PowerPointExtractor pptExtractor = new PowerPointExtractor(pptStream);
            Reader contentReader = new StringReader(pptExtractor.getText(true, true));
            pptStream.close();
            addDoc(indexWriter, contentReader, fullName);
            pptExtractor.close();
            contentReader.close();
        } else if (extension.equals(".txt")) {
            //Text Document
            Reader contentReader = new BufferedReader(new FileReader(pathName));
            addDoc(indexWriter, contentReader, fullName);
            contentReader.close();
        } else if (extension.equals(".xls")) {
            //MSExcelExtractor Document
            //InputStream excelStream=new BufferedInputStream(new FileInputStream(pathName));
            //ExcelExtractor excelExtractor= new ExcelExtractor(excelStream);
            //Reader contentReader=new StringReader(excelExtractor.getText());
            //excelStream.close();
            //addDoc(indexWriter,contentReader,fullName);
            //excelExtractor.close();
            //contentReader.close();
        } else if (extension.equals(".html") || extension.equals(".htm")) {
        } else if (extension.equals(".csv")) {
        } else if (extension.equals(".xml")) {
        } else if (extension.equals(".rtf")) {
        } else if (extension.equals(".pdf")) {
        } else if (extension.equals(".msg")) {
        }
    } catch (CorruptIndexException ex) {
        throw new EJBException(ex);
    } catch (LockObtainFailedException ex) {
        try {
            if (IndexWriter.isLocked(indexDir)) {
                IndexWriter.unlock(indexDir);
            }
        } catch (IOException pIOEx) {
            throw new EJBException(pIOEx);
        }
        throw new EJBException(ex);
    } catch (ParserConfigurationException ex) {
        throw new EJBException(ex);
    } catch (SAXException ex) {
        throw new EJBException(ex);
    } catch (IOException ex) {
        throw new EJBException(ex);
    } finally {
        try {
            if (indexWriter != null) {
                indexWriter.close();
            }
        } catch (IOException ex) {
            throw new EJBException(ex);
        }
    }
}

From source file:com.doculibre.constellio.lucene.BaseLuceneIndexHelper.java

License:Open Source License

@Override
public synchronized void add(T object) {
    try {/*from ww w  .  j ava  2s . c o m*/
        Directory directory = FSDirectory.open(indexDir);
        Analyzer analyzer = analyzerProvider.getAnalyzer(Locale.FRENCH);
        IndexWriter indexWriter = new IndexWriter(directory,
                new IndexWriterConfig(Version.LUCENE_44, analyzer));
        add(object, indexWriter);
        indexWriter.close();
        directory.close();
    } catch (IOException e) {
        throw new RuntimeException(e);
    }
}