Example usage for org.apache.lucene.index IndexWriter abortMerges

Introduction

In this page you can find the example usage for org.apache.lucene.index IndexWriter abortMerges.

Prototype

private synchronized void abortMerges() 

Source Link

Document

Aborts running merges.

Usage

From source file:perf.IndexNumbers.java

License:Apache License

public static void main(String[] args) throws Exception {
    String numbersFile = args[0];
    File indexPath = new File(args[1]);
    int numThreads = Integer.parseInt(args[2]);
    int precStep = Integer.parseInt(args[3]);
    if (indexPath.exists()) {
        throw new IllegalArgumentException("please remove indexPath \"" + indexPath + "\" before running");
    }/*from w  w  w.  ja v a 2s .  c  o  m*/

    Directory dir = FSDirectory.open(indexPath);
    IndexWriterConfig iwc = new IndexWriterConfig(new StandardAnalyzer());
    //iwc.setRAMBufferSizeMB(350);
    //iwc.setInfoStream(new PrintStreamInfoStream(System.out));
    if (normal == false) {
        iwc.setRAMBufferSizeMB(64);
        iwc.setMergePolicy(NoMergePolicy.INSTANCE);
    } else {
        // 5/5 segments:
        // 3273681 docs in twitter timestamps
        iwc.setMaxBufferedDocs(59522);
        iwc.setRAMBufferSizeMB(-1);
    }
    //((ConcurrentMergeScheduler) iwc.getMergeScheduler()).setMaxMergesAndThreads(3, 1);
    final IndexWriter w = new IndexWriter(dir, iwc);

    final Field.Store store = Field.Store.NO;

    final FieldType doubleFieldType = new FieldType(
            store == Field.Store.NO ? DoubleField.TYPE_NOT_STORED : DoubleField.TYPE_STORED);
    doubleFieldType.setNumericPrecisionStep(precStep);
    doubleFieldType.freeze();

    final FieldType longFieldType = new FieldType(
            store == Field.Store.NO ? LongField.TYPE_NOT_STORED : LongField.TYPE_STORED);
    longFieldType.setNumericPrecisionStep(precStep);
    longFieldType.freeze();

    final FieldType intFieldType = new FieldType(
            store == Field.Store.NO ? IntField.TYPE_NOT_STORED : IntField.TYPE_STORED);
    intFieldType.setNumericPrecisionStep(precStep);
    intFieldType.freeze();

    // 64K buffer:
    InputStream is = new FileInputStream(numbersFile);
    final BufferedReader reader = new BufferedReader(new InputStreamReader(is, "UTF-8"), 1 << 16);
    final AtomicInteger docsIndexed = new AtomicInteger();

    final long startMS = System.currentTimeMillis();
    Thread[] threads = new Thread[numThreads];
    for (int i = 0; i < numThreads; i++) {
        threads[i] = new Thread() {
            @Override
            public void run() {
                ParsePosition datePos = new ParsePosition(0);
                SimpleDateFormat dateParser = new SimpleDateFormat("yyyy-MM-dd", Locale.US);

                Document doc = new Document();
                Field field = new LongField("number", 0L, longFieldType);
                doc.add(field);

                while (true) {
                    try {
                        // Curiously BufferedReader.readLine seems to be thread-safe...
                        String line = reader.readLine();
                        if (line == null) {
                            break;
                        }
                        field.setLongValue(Long.parseLong(line.trim()));
                        w.addDocument(doc);
                        int count = docsIndexed.incrementAndGet();
                        if (count % 200000 == 0) {
                            long ms = System.currentTimeMillis();
                            System.out.println(count + ": " + ((ms - startMS) / 1000.0) + " sec");
                        }
                    } catch (Exception e) {
                        throw new RuntimeException(e);
                    }
                }
            }
        };
        threads[i].start();
    }
    for (int i = 0; i < numThreads; i++) {
        threads[i].join();
    }
    long ms = System.currentTimeMillis();
    System.out.println(docsIndexed + ": " + ((ms - startMS) / 1000.0) + " sec");
    //System.out.println("tot conflicts: " + BytesRefHash.totConflict);
    if (normal == false) {
        w.abortMerges();
    }
    w.close();
    dir.close();
}

---