Example usage for org.apache.lucene.codecs.compressing CompressionMode HIGH_COMPRESSION

Introduction

In this page you can find the example usage for org.apache.lucene.codecs.compressing CompressionMode HIGH_COMPRESSION.

Prototype

CompressionMode HIGH_COMPRESSION

To view the source code for org.apache.lucene.codecs.compressing CompressionMode HIGH_COMPRESSION.

Click Source Link

Document

A compression mode that trades speed for compression ratio.

Usage

From source file:com.lucenetutorial.apps.CompressingCodec.java

public MyStoredFieldFormat() {
    super("MyStoredFieldFormat", CompressionMode.HIGH_COMPRESSION, 1 << 28, 1 << 28, 1 << 27);
    // super(null, CompressionMode.FAST, chunkSize, maxDocsPerChunk, blockSize)
    //For a chunk size of chunkSize bytes, this StoredFieldsFormat does not support documents larger than (2^31 - chunkSize) bytes.

}

---

From source file:org.apache.blur.lucene.codec.Blur022Codec.java

License:Apache License

private static CompressionMode getCompressionMode(BlurConfiguration configuration) {
    String type = configuration.get(BLUR_SHARD_INDEX_COMPRESSIONMODE, FAST);
    if (HIGH_COMPRESSION.equals(type)) {
        return CompressionMode.HIGH_COMPRESSION;
    } else if (FAST.equals(type)) {
        return CompressionMode.FAST;
    } else if (FAST_DECOMPRESSION.equals(type)) {
        return CompressionMode.FAST_DECOMPRESSION;
    } else {//from   ww w.j  a  v  a  2  s . c  o m
        throw new IllegalArgumentException("blur.shard.index.compressionmode=" + type
                + " not supported.  Valid entries are [FAST,FAST_DECOMPRESSION,HIGH_COMPRESSION]");
    }
}

---

From source file:org.apache.blur.lucene.codec.Blur022CodecTest.java

License:Apache License

@Test
public void testLargeDocs() throws IOException {
    Random random = new Random();
    Iterable<? extends IndexableField> doc = getLargeDoc(random);
    RAMDirectory directory = new RAMDirectory();
    IndexWriterConfig conf1 = new IndexWriterConfig(Version.LUCENE_43,
            new WhitespaceAnalyzer(Version.LUCENE_43));
    conf1.setCodec(new Blur022Codec());
    IndexWriter writer1 = new IndexWriter(directory, conf1);
    writer1.addDocument(doc);/*from w w w  .  j  a v a2s.  co m*/
    writer1.close();

    DirectoryReader reader1 = DirectoryReader.open(directory);
    int numDocs1 = reader1.numDocs();
    assertEquals(1, numDocs1);

    // for (int i = 0; i < numDocs1; i++) {
    // System.out.println(reader1.document(i));
    // }

    IndexWriterConfig conf2 = new IndexWriterConfig(Version.LUCENE_43,
            new WhitespaceAnalyzer(Version.LUCENE_43));
    conf2.setCodec(new Blur022Codec(1 << 16, CompressionMode.HIGH_COMPRESSION));
    IndexWriter writer2 = new IndexWriter(directory, conf2);
    writer2.addDocument(doc);
    writer2.close();

    DirectoryReader reader2 = DirectoryReader.open(directory);
    int numDocs2 = reader2.numDocs();
    assertEquals(2, numDocs2);

    for (int i = 0; i < 2; i++) {

        long t1 = System.nanoTime();
        Document document1 = reader1.document(0);
        long t2 = System.nanoTime();
        Document document2 = reader2.document(1);
        long t3 = System.nanoTime();

        System.out.println((t3 - t2) / 1000000.0);
        System.out.println((t2 - t1) / 1000000.0);

        System.out.println("doc1 " + document1.hashCode());
        System.out.println("doc2 " + document2.hashCode());
    }

    // for (int i = 0; i < numDocs2; i++) {
    // System.out.println(reader2.document(i));
    // }

    // long fileLength = directory.fileLength("_0.fdt");

    for (String name : directory.listAll()) {
        if (name.endsWith(".fdt")) {
            System.out.println(name);
            System.out.println(directory.fileLength(name));
        }
    }

}

---

From source file:org.apache.blur.lucene.codec.Blur022CodecTest.java

License:Apache License

@Test
public void testSmallDocs() throws IOException {

    RAMDirectory directory = new RAMDirectory();
    IndexWriterConfig conf1 = new IndexWriterConfig(Version.LUCENE_43,
            new WhitespaceAnalyzer(Version.LUCENE_43));
    conf1.setCodec(new Blur022Codec());
    Random random1 = new Random(1);
    IndexWriter writer1 = new IndexWriter(directory, conf1);
    for (int i = 0; i < 1000; i++) {
        writer1.addDocument(getSmallDoc(random1));
    }/* www .j ava 2  s. com*/
    writer1.close();

    DirectoryReader reader1 = DirectoryReader.open(directory);
    int numDocs1 = reader1.numDocs();
    assertEquals(1000, numDocs1);

    // for (int i = 0; i < numDocs1; i++) {
    // System.out.println(reader1.document(i));
    // }

    IndexWriterConfig conf2 = new IndexWriterConfig(Version.LUCENE_43,
            new WhitespaceAnalyzer(Version.LUCENE_43));
    conf2.setCodec(new Blur022Codec(1 << 16, CompressionMode.HIGH_COMPRESSION));
    Random random2 = new Random(1);
    IndexWriter writer2 = new IndexWriter(directory, conf2);
    for (int i = 0; i < 1000; i++) {
        writer2.addDocument(getSmallDoc(random2));
    }
    writer2.close();

    DirectoryReader reader2 = DirectoryReader.open(directory);
    int numDocs2 = reader2.numDocs();
    assertEquals(2000, numDocs2);

    for (int i = 0; i < 2; i++) {

        long t1 = System.nanoTime();
        long hash1 = 0;
        long hash2 = 0;
        for (int d = 0; d < 1000; d++) {
            Document document1 = reader1.document(d);
            hash1 += document1.hashCode();
        }
        long t2 = System.nanoTime();
        for (int d = 0; d < 1000; d++) {
            Document document2 = reader2.document(d + 1000);
            hash2 += document2.hashCode();
        }
        long t3 = System.nanoTime();

        System.out.println((t3 - t2) / 1000000.0);
        System.out.println((t2 - t1) / 1000000.0);

        System.out.println("doc1 " + hash1);
        System.out.println("doc2 " + hash2);
    }

    // for (int i = 0; i < numDocs2; i++) {
    // System.out.println(reader2.document(i));
    // }

    // long fileLength = directory.fileLength("_0.fdt");

    for (String name : directory.listAll()) {
        if (name.endsWith(".fdt")) {
            System.out.println(name);
            System.out.println(directory.fileLength(name));
        }
    }
}

---

From source file:org.apache.blur.lucene.codec.Blur022StoredFieldsFormat.java

License:Apache License

private CompressionMode getCompressionMode(SegmentInfo si) {
    String attribute = si.getAttribute(STORED_FIELDS_FORMAT_COMPRESSION_MODE);
    if (HIGH_COMPRESSION.equals(attribute)) {
        return CompressionMode.HIGH_COMPRESSION;
    } else if (FAST.equals(attribute)) {
        return CompressionMode.FAST;
    } else if (FAST_DECOMPRESSION.equals(attribute)) {
        return CompressionMode.FAST_DECOMPRESSION;
    }/*from w w  w  .jav  a 2s .  co m*/
    // This happen during nrt udpates.
    return _compressionMode;
}

---

From source file:org.apache.blur.lucene.codec.Blur024CodecTest.java

License:Apache License

@Test
public void testLargeDocs() throws IOException {
    Random random = new Random();
    Iterable<? extends IndexableField> doc = getLargeDoc(random);
    RAMDirectory directory = new RAMDirectory();
    IndexWriterConfig conf1 = new IndexWriterConfig(Version.LUCENE_43,
            new WhitespaceAnalyzer(Version.LUCENE_43));
    conf1.setCodec(new Blur024Codec());
    IndexWriter writer1 = new IndexWriter(directory, conf1);
    writer1.addDocument(doc);//ww w.j av  a  2 s  .  co  m
    writer1.close();

    DirectoryReader reader1 = DirectoryReader.open(directory);
    int numDocs1 = reader1.numDocs();
    assertEquals(1, numDocs1);

    // for (int i = 0; i < numDocs1; i++) {
    // System.out.println(reader1.document(i));
    // }

    IndexWriterConfig conf2 = new IndexWriterConfig(Version.LUCENE_43,
            new WhitespaceAnalyzer(Version.LUCENE_43));
    conf2.setCodec(new Blur024Codec(1 << 16, CompressionMode.HIGH_COMPRESSION));
    IndexWriter writer2 = new IndexWriter(directory, conf2);
    writer2.addDocument(doc);
    writer2.close();

    DirectoryReader reader2 = DirectoryReader.open(directory);
    int numDocs2 = reader2.numDocs();
    assertEquals(2, numDocs2);

    for (int i = 0; i < 2; i++) {

        long t1 = System.nanoTime();
        Document document1 = reader1.document(0);
        long t2 = System.nanoTime();
        Document document2 = reader2.document(1);
        long t3 = System.nanoTime();

        System.out.println((t3 - t2) / 1000000.0);
        System.out.println((t2 - t1) / 1000000.0);

        System.out.println("doc1 " + document1.hashCode());
        System.out.println("doc2 " + document2.hashCode());
    }

    // for (int i = 0; i < numDocs2; i++) {
    // System.out.println(reader2.document(i));
    // }

    // long fileLength = directory.fileLength("_0.fdt");

    for (String name : directory.listAll()) {
        if (name.endsWith(".fdt")) {
            System.out.println(name);
            System.out.println(directory.fileLength(name));
        }
    }

}

---

From source file:org.apache.blur.lucene.codec.Blur024CodecTest.java

License:Apache License

@Test
public void testSmallDocs() throws IOException {

    RAMDirectory directory = new RAMDirectory();
    IndexWriterConfig conf1 = new IndexWriterConfig(Version.LUCENE_43,
            new WhitespaceAnalyzer(Version.LUCENE_43));
    conf1.setCodec(new Blur024Codec());
    Random random1 = new Random(1);
    IndexWriter writer1 = new IndexWriter(directory, conf1);
    for (int i = 0; i < 1000; i++) {
        writer1.addDocument(getSmallDoc(random1));
    }//from   www. j  a v  a 2  s .c om
    writer1.close();

    DirectoryReader reader1 = DirectoryReader.open(directory);
    int numDocs1 = reader1.numDocs();
    assertEquals(1000, numDocs1);

    // for (int i = 0; i < numDocs1; i++) {
    // System.out.println(reader1.document(i));
    // }

    IndexWriterConfig conf2 = new IndexWriterConfig(Version.LUCENE_43,
            new WhitespaceAnalyzer(Version.LUCENE_43));
    conf2.setCodec(new Blur024Codec(1 << 16, CompressionMode.HIGH_COMPRESSION));
    Random random2 = new Random(1);
    IndexWriter writer2 = new IndexWriter(directory, conf2);
    for (int i = 0; i < 1000; i++) {
        writer2.addDocument(getSmallDoc(random2));
    }
    writer2.close();

    DirectoryReader reader2 = DirectoryReader.open(directory);
    int numDocs2 = reader2.numDocs();
    assertEquals(2000, numDocs2);

    for (int i = 0; i < 2; i++) {

        long t1 = System.nanoTime();
        long hash1 = 0;
        long hash2 = 0;
        for (int d = 0; d < 1000; d++) {
            Document document1 = reader1.document(d);
            hash1 += document1.hashCode();
        }
        long t2 = System.nanoTime();
        for (int d = 0; d < 1000; d++) {
            Document document2 = reader2.document(d + 1000);
            hash2 += document2.hashCode();
        }
        long t3 = System.nanoTime();

        System.out.println((t3 - t2) / 1000000.0);
        System.out.println((t2 - t1) / 1000000.0);

        System.out.println("doc1 " + hash1);
        System.out.println("doc2 " + hash2);
    }

    // for (int i = 0; i < numDocs2; i++) {
    // System.out.println(reader2.document(i));
    // }

    // long fileLength = directory.fileLength("_0.fdt");

    for (String name : directory.listAll()) {
        if (name.endsWith(".fdt")) {
            System.out.println(name);
            System.out.println(directory.fileLength(name));
        }
    }
}

---