List of usage examples for org.apache.lucene.index IndexWriter addDocuments
public long addDocuments(Iterable<? extends Iterable<? extends IndexableField>> docs) throws IOException
From source file:org.apache.blur.lucene.search.RandomSuperQueryTest.java
License:Apache License
private Directory createIndex(Random random, Collection<Query> sampler) throws CorruptIndexException, LockObtainFailedException, IOException { Directory directory = new RAMDirectory(); String[] columnFamilies = genWords(random, MIN_NUM_COL_FAM, MAX_NUM_COL_FAM, "colfam"); Map<String, String[]> columns = new HashMap<String, String[]>(); for (int i = 0; i < columnFamilies.length; i++) { columns.put(columnFamilies[i], genWords(random, MIN_NUM_COLS, MAX_NUM_COLS, "col")); }// w w w . j ava 2s. c om IndexWriter writer = new IndexWriter(directory, new IndexWriterConfig(LUCENE_VERSION, new StandardAnalyzer(LUCENE_VERSION))); int numberOfDocs = random.nextInt(MAX_NUM_OF_DOCS) + 1; for (int i = 0; i < numberOfDocs; i++) { writer.addDocuments(generatSuperDoc(random, columns, sampler)); } writer.close(); return directory; }
From source file:org.apache.blur.lucene.search.SuperQueryTest.java
License:Apache License
public static Directory createIndex() throws CorruptIndexException, LockObtainFailedException, IOException { Directory directory = new RAMDirectory(); IndexWriter writer = new IndexWriter(directory, new IndexWriterConfig(LUCENE_VERSION, new StandardAnalyzer(LUCENE_VERSION))); writer.addDocuments( addPrime(Arrays.asList(newDocument(newStringField(ROW_ID, "1"), newStringField(PERSON_NAME, NAME1)), newDocument(newStringField(ROW_ID, "1"), newStringField(PERSON_NAME, NAME1)), newDocument(newStringField(ROW_ID, "1"), newStringField(ADDRESS_STREET, STREET1))))); writer.addDocuments(//from w w w . ja va2s. co m addPrime(Arrays.asList(newDocument(newStringField(ROW_ID, "2"), newStringField(PERSON_NAME, NAME2)), newDocument(newStringField(ROW_ID, "2"), newStringField(ADDRESS_STREET, STREET1))))); writer.addDocuments( addPrime(Arrays.asList(newDocument(newStringField(ROW_ID, "3"), newStringField(PERSON_NAME, NAME1)), newDocument(newStringField(ROW_ID, "3"), newStringField(ADDRESS_STREET, STREET1)), newDocument(newStringField(ROW_ID, "3"), newStringField(ADDRESS_STREET, STREET2))))); writer.close(); return directory; }
From source file:org.elasticsearch.common.lucene.search.XBooleanFilterTests.java
License:Apache License
@Before public void setup() throws Exception { super.setUp(); char[][] documentMatrix = new char[][] { { 'a', 'b', 'c', 'd', 'v' }, { 'a', 'b', 'c', 'd', 'z' }, { 'a', 'a', 'a', 'a', 'x' } }; List<Document> documents = new ArrayList<Document>(documentMatrix.length); for (char[] fields : documentMatrix) { Document document = new Document(); for (int i = 0; i < fields.length; i++) { document.add(new StringField(Integer.toString(i), String.valueOf(fields[i]), Field.Store.NO)); }/*ww w. j av a 2 s .co m*/ documents.add(document); } directory = newDirectory(); IndexWriter w = new IndexWriter(directory, new IndexWriterConfig(Lucene.VERSION, new KeywordAnalyzer())); w.addDocuments(documents); w.close(); reader = SlowCompositeReaderWrapper.wrap(DirectoryReader.open(directory)); }
From source file:org.hibernate.search.backend.lucene.work.impl.AddEntryLuceneWork.java
License:LGPL
private CompletableFuture<Long> addEntry(IndexWriter indexWriter) { try {//from w ww. jav a 2 s. c o m return CompletableFuture.completedFuture(indexWriter.addDocuments(indexEntry)); } catch (IOException e) { throw log.unableToIndexEntry(tenantId, id, getEventContext(), e); } }
From source file:org.hibernate.search.backend.lucene.work.impl.QueryBasedUpdateEntryLuceneWork.java
License:LGPL
@Override protected long doUpdateEntry(IndexWriter indexWriter, String tenantId, String id, LuceneIndexEntry indexEntry) throws IOException { indexWriter.deleteDocuments(LuceneQueries.discriminatorMultiTenancyDeleteDocumentQuery(tenantId, id)); return indexWriter.addDocuments(indexEntry); }
From source file:org.lexevs.dao.index.lucenesupport.BaseLuceneIndexTemplate.java
License:Open Source License
public void addDocuments(final List<Document> documents, final Analyzer analyzer) { this.doInIndexWriter(new IndexWriterCallback<Void>() { @Override// w w w. jav a2s .com public Void doInIndexWriter(IndexWriter indexWriter) throws Exception { indexWriter.addDocuments(documents); return null; } }); }
From source file:org.lexevs.dao.index.lucenesupport.MetaDataIndexTemplate.java
License:Open Source License
@Override public void addDocuments(final List<Document> documents, final Analyzer analyzer) { this.doInIndexWriter(new IndexWriterCallback<Void>() { @Override//from ww w . j a v a 2s . c o m public Void doInIndexWriter(IndexWriter indexWriter) throws Exception { indexWriter.addDocuments(documents); return null; } }); }
From source file:org.silverpeas.search.indexEngine.model.TestDidYouMeanIndexer.java
License:Open Source License
/** * @throws java.lang.Exception//from w w w . j a v a 2 s . com */ @BeforeClass public static void setUpBeforeClass() throws Exception { File originalIndexes = new File(originalIndexpath); if (originalIndexes.exists()) { FileUtils.forceDelete(originalIndexes); } originalIndexes.mkdirs(); FSDirectory directory = FSDirectory.open(originalIndexes); IndexWriterConfig configuration = new IndexWriterConfig(Version.LUCENE_36, new StandardAnalyzer(Version.LUCENE_36)); IndexWriter writer = new IndexWriter(directory, configuration); List<Document> docs = new ArrayList<Document>(6); docs.add(createDocument("2010/01/25", "0", "9999/99/99", "kmelia1|Node|3", null, "", "0000/00/00", "J2EE", new StringReader(("j2ee J2EE administrateur j2ee")))); docs.add(createDocument("2010/01/25", "0", "9999/99/99", "kmelia1|Publication|1", null, "", "2010/01/25", "Introduction au Java Framework", new StringReader(("1 introduction java framework Introduction au Java Framework " + "null administrateur java framework java" + "2 platform est compose trois " + "editions destinees usages differents j2me " + "java 2 micro edition est " + "prevu pour developpement applications embarquees " + "notamment sur assistants personnels terminaux " + "mobiles j2se java 2 standard " + "edition est destine developpement applications " + "pour ordinateurs personnels j2ee java " + "2 enterprise edition destine usage " + "professionnel avec mise en oeuvre " + "serveurs chaque edition propose environnement " + "complet pour developpement execution applications " + "basees sur java comprend notamment " + "machine virtuelle java java virtual " + "machine ainsi un ensemble classes " + "introduction java framework null")))); docs.add(createDocument("2010/01/25", "0", "9999/99/99", "kmelia1|Attachment1|1", null, null, "0000/00/00", "1wysiwyg.txt", new StringReader("1wysiwyg.txt java framework java 2 " + "platform est compose trois editions " + "destinees usages differents j2me java " + "2 micro edition est prevu " + "pour developpement applications embarquees notamment " + "sur assistants personnels terminaux mobiles " + "j2se java 2 standard edition " + "est destine developpement applications pour " + "ordinateurs personnels j2ee java 2 " + "enterprise edition destine usage professionnel " + "avec mise en oeuvre serveurs " + "chaque edition propose environnement complet " + "pour developpement execution applications basees " + "sur java comprend notamment machine " + "virtuelle java java virtual machine " + "ainsi un ensemble classes 1wysiwyg.txt"))); docs.add(createDocument("2010/01/25", "0", "9999/99/99", "kmelia1|Publication|2", "java J2ee introduction", "", "2010/01/25", "Introduction J2EE ", new StringReader("2 introduction j2ee Introduction J2EE java " + "j2ee introduction null administrateur j2ee " + "java 2 enterprise edition est " + "norme proposee par societe sun " + "portee par consortium societes internationales " + "visant definir standard developpement applications " + "entreprises multi niveaux basees sur " + "composants on parle generalement plate " + "forme j2ee pour designer ensemble " + "constitue services api offerts infrastructure " + "execution j2ee comprend notamment specifications " + "serveur application est dire environnement " + "execution j2ee definit finement roles " + "interfaces pour applications ainsi environnement " + "dans seront executees ces recommandations " + "permettent ainsi entreprises tierces developper " + "serveurs application conformes specifications ainsi " + "definies sans avoir redevelopper principaux " + "services services travers api est " + "dire extensions java independantes permettant " + "offrir en standard certain nombre " + "fonctionnalites sun fournit implementation minimale " + "ces api appelee j2ee sdk " + "j2ee software development kit dans " + "mesure j2ee appuie entierement sur " + "java beneficie avantages inconvenients ce " + "langage en particulier bonne portabilite " + "maintenabilite code plus architecture j2ee " + "repose sur composants distincts interchangeables " + "distribues ce qui signifie notamment " + "il est simple etendre architecture " + "un systeme reposant sur j2ee " + "peut posseder mecanismes haute disponibilite " + "afin garantir bonne qualite service " + "maintenabilite applications est facilitee introduction " + "j2ee java j2ee introduction null"))); docs.add(createDocument("2010/01/25", "0", "9999/99/99", "kmelia1|Attachment2|2", null, null, "0000/00/00", "2wysiwyg.txt", new StringReader("2wysiwyg.txt j2ee java 2 enterprise " + "edition est norme proposee par " + "societe sun portee par consortium " + "societes internationales visant definir standard " + "developpement applications entreprises multi niveaux " + "basees sur composants on parle " + "generalement plate forme j2ee pour " + "designer ensemble constitue services api " + "offerts infrastructure execution j2ee comprend " + "notamment specifications serveur application est " + "dire environnement execution j2ee definit " + "finement roles interfaces pour applications " + "ainsi environnement dans seront executees " + "ces recommandations permettent ainsi entreprises " + "tierces developper serveurs application conformes " + "specifications ainsi definies sans avoir " + "redevelopper principaux services services travers " + "api est dire extensions java "))); docs.add(createDocument("2010/01/27", "0", "9999/99/99", "kmelia1|Attachment7|3", null, null, "0000/00/00", "3wysiwyg.txt", new StringReader("3wysiwyg.txt api j2ee peuvent se " + "repartir en trois grandes categories " + "composants on distingue habituellement deux " + "familles composants composants web servlets " + "jsp java server pages agit " + "partie chargee interface avec utilisateur " + "on parle logique presentation composants " + "metier ejb enterprise java beans " + "agit composants specifiques charges traitements " + "donnees propres secteur activite on " + "parle logique metier logique applicative " + "interfacage avec bases donnees services " + "pouvant etre classes par categories " + "services infrastructures en existe grand " + "nombre definis ci dessous jdbc " + "java database connectivity est api " + "acces bases donnees relationnelles jndi " + "java naming directory interface est " + "api acces services nommage annuaires " + "entreprises tels dns nis ldap " + "etc jta jts java transaction " + "api java transaction services est " + "api definissant interfaces standard avec " + "gestionnaire transactions jca j2ee connector " + "architecture est api connexion systeme " + "information entreprise notamment systemes dits " + "legacy tels erp jmx java " + "management extension fournit extensions permettant " + "developper applications web supervision applications " + "services communication jaas java authentication " + "authorization service est api gestion " + "authentification droits acces javamail est " + "api permettant envoi courrier electronique " + "jms java message service fournit " + "fonctionnalites communication asynchrone appelees mom " + "pour middleware object message entre " + "applications rmi iiop est api " + "permettant communication synchrone entre objets " + "architecture j2ee permet ainsi separer " + "couche presentation correspondant interface homme " + "machine ihm couche metier contenant " + "essentiel traitements donnees en se " + "basant dans mesure possible sur " + "api existantes enfin couche donnees " + "correspondant informations entreprise stockees dans " + "fichiers dans bases donnees relationnelles " + "xml dans annuaires entreprise encore " + "dans systemes information complexes 3wysiwyg.txt"))); docs.add(createDocument("2010/01/27", "0", "9999/99/99", "kmelia1|Publication|3", "", "", "2010/01/25", "Les API de J2EE", new StringReader("3 api j2ee Les API de J2EE administrateur " + "api j2ee peuvent se repartir " + "en trois grandes categories composants " + "on distingue habituellement deux familles " + "composants composants web servlets jsp " + "java server pages agit partie " + "chargee interface avec utilisateur on " + "parle logique presentation composants metier " + "ejb enterprise java beans agit " + "composants specifiques charges traitements donnees " + "propres secteur activite on parle " + "logique metier logique applicative interfacage " + "avec bases donnees services pouvant " + "etre classes par categories services " + "infrastructures en existe grand nombre " + "definis ci dessous jdbc java " + "database connectivity est api acces " + "bases donnees relationnelles jndi java " + "naming directory interface est api " + "acces services nommage annuaires entreprises " + "tels dns nis ldap etc " + "jta jts java transaction api " + "java transaction services est api " + "definissant interfaces standard avec gestionnaire " + "transactions jca j2ee connector architecture " + "est api connexion systeme information " + "entreprise notamment systemes dits legacy " + "tels erp jmx java management " + "extension fournit extensions permettant developper " + "applications web supervision applications services " + "communication jaas java authentication authorization " + "service est api gestion authentification " + "droits acces javamail est api " + "permettant envoi courrier electronique jms " + "java message service fournit fonctionnalites " + "communication asynchrone appelees mom pour " + "middleware object message entre applications " + "rmi iiop est api permettant " + "communication synchrone entre objets architecture " + "j2ee permet ainsi separer couche " + "presentation correspondant interface homme machine " + "ihm couche metier contenant essentiel " + "traitements donnees en se basant " + "dans mesure possible sur api " + "existantes enfin couche donnees correspondant " + "informations entreprise stockees dans fichiers " + "dans bases donnees relationnelles xml " + "dans annuaires entreprise encore dans " + "systemes information complexes api j2ee"))); writer.addDocuments(docs); writer.close(); }
From source file:uk.ac.ebi.mdk.service.loader.single.MoleculeCollectionConnectivityLoader.java
License:Open Source License
@Override public void update() throws IOException { LinkedList<Document> docs = new LinkedList(); MoleculeConnectivity entry;//from www. j av a 2 s . com int counter = 0; while (molIterator.hasNext()) { entry = molIterator.next(); if (checkEntry(entry)) { counter++; Document doc = new Document(); doc.add(new Field(MoleculeCollectionConnectivityLuceneFields.CollectionName.toString(), this.collectionName, Field.Store.YES, Field.Index.NOT_ANALYZED)); doc.add(new Field(MoleculeCollectionConnectivityLuceneFields.Identifier.toString(), entry.getId(), Field.Store.YES, Field.Index.NOT_ANALYZED)); doc.add(new Field(MoleculeCollectionConnectivityLuceneFields.DB.toString(), entry.getDB(), Field.Store.YES, Field.Index.NOT_ANALYZED)); doc.add(new Field(MoleculeCollectionConnectivityLuceneFields.Connectivity.toString(), entry.getConnectivity(), Field.Store.YES, Field.Index.NOT_ANALYZED)); docs.add(doc); if (counter % 500 == 0) { LOGGER.info("Indexed " + counter + " mols."); } } else { LOGGER.warn("Could not add entry " + entry.getId() + " " + entry.getDB() + " " + entry.getConnectivity()); } } // write the index Directory index = getIndex().getDirectory(); IndexWriter writer = new IndexWriter(index, new IndexWriterConfig(Version.LUCENE_34, getIndex().getAnalyzer())); writer.addDocuments(docs); writer.close(); index.close(); }
From source file:Util.Index_Handler.java
/** * Creates a block index . Is necessary for blockjoinquerys. * @return status of operation/*w ww.j a v a2 s .c o m*/ */ public static String createBlockIndex() { try { Scanner scan = new Scanner(Paths.get("combined.txt")); Scanner scan2 = new Scanner(Paths.get("entity_anchors.txt")); Analyzer analyzer = new StandardAnalyzer(Version.LUCENE_46); Directory dir = FSDirectory.open(new File("Entity_Index")); IndexWriterConfig conf = new IndexWriterConfig(Version.LUCENE_46, analyzer); conf.setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND); conf.setSimilarity(new CustomSimilarity()); IndexWriter writer = new IndexWriter(dir, conf); String[] line;//=scan.nextLine().split(" "); ArrayList<Document> group = new ArrayList<>(); Map<String, String> anchorMap = new HashMap<>(); Document doc; String[] anchors; //load anchors into ram while (scan2.hasNext()) { line = scan2.nextLine().split("\\|"); anchorMap.put(line[0], line[1]); } line = scan.nextLine().split("\\|"); String previous = line[0]; anchors = line[2].split(";"); for (String s : anchors) { doc = new Document(); doc.add(new Field("anchorN", s, TextField.TYPE_STORED)); group.add(doc); } while (scan.hasNext()) { line = scan.nextLine().split("\\|"); if (line.length == 0) { System.out.println("found end"); doc = new Document(); doc.add(new Field("entity", line[0], TextField.TYPE_STORED)); doc.add(new Field("anchor", anchorDeli(anchorMap.get(line[0])), TextField.TYPE_STORED)); doc.add(new Field("titel", delimeterString(getEntity(line[0])), TextField.TYPE_STORED)); //this is the marker for the parent field. // it needs to be a stringfield so only an exact match will hit // and does not need to be in the index since you dont search this field. doc.add(new Field("type", "Parent", StringField.TYPE_NOT_STORED)); group.add(doc); writer.addDocuments(group); group.clear(); break; } if (!line[0].equals(previous)) { doc = new Document(); doc.add(new Field("entity", line[0], TextField.TYPE_STORED)); doc.add(new Field("anchor", anchorDeli(anchorMap.get(line[0])), TextField.TYPE_STORED)); doc.add(new Field("title", delimeterString(getEntity(line[0])), TextField.TYPE_STORED)); //this is the marker for the parent field. // it needs to be a stringfield so only an exact match will hit // and does not need to be in the index since you dont search this field. doc.add(new Field("type", "Parent", StringField.TYPE_NOT_STORED)); group.add(doc); writer.addDocuments(group); group.clear(); previous = line[0]; } anchors = line[2].split(";"); for (String s : anchors) { doc = new Document(); doc.add(new Field("anchorN", s, TextField.TYPE_STORED)); group.add(doc); } } writer.prepareCommit(); writer.commit(); writer.close(); } catch (IOException ex) { System.out.println("Creating Blockindex failed :" + ex.getMessage()); return "Creating Blockindex failed :" + ex.getMessage(); } return "done"; }