Example usage for org.apache.lucene.index IndexReader close

Introduction

In this page you can find the example usage for org.apache.lucene.index IndexReader close.

Prototype

@Override
public final synchronized void close() throws IOException

Source Link

Document

Closes files associated with this index.

Usage

From source file:com.searchcode.app.service.CodeSearcher.java

License:Open Source License

/**
 * Only used as fallback if getByRepoFileName fails for some reason due to what appears to be a lucene index bug
 * this should always work as the path used is sha1 and should be unique for anything the current codebase can
 * deal with//from  ww  w  .  j av  a  2  s  .co m
 */
public CodeResult getByCodeId(String codeId) {
    CodeResult codeResult = null;

    try {
        IndexReader reader = DirectoryReader.open(FSDirectory.open(Paths.get(this.INDEXPATH)));
        IndexSearcher searcher = new IndexSearcher(reader);
        Analyzer analyzer = new CodeAnalyzer();
        QueryParser parser = new QueryParser(CODEFIELD, analyzer);

        Query query = parser.parse(Values.CODEID + ":" + QueryParser.escape(codeId));
        Singleton.getLogger().info("Query to get by " + Values.CODEID + ":" + QueryParser.escape(codeId));

        TopDocs results = searcher.search(query, 1);
        ScoreDoc[] hits = results.scoreDocs;

        if (hits.length != 0) {
            Document doc = searcher.doc(hits[0].doc);

            String filepath = doc.get(Values.PATH);

            List<String> code = new ArrayList<>();
            try {
                code = Singleton.getHelpers()
                        .readFileLinesGuessEncoding(filepath,
                                Singleton.getHelpers().tryParseInt(
                                        Properties.getProperties().getProperty(Values.MAXFILELINEDEPTH,
                                                Values.DEFAULTMAXFILELINEDEPTH),
                                        Values.DEFAULTMAXFILELINEDEPTH));
            } catch (Exception ex) {
                Singleton.getLogger().info("Indexed file appears to binary: " + filepath);
            }

            codeResult = new CodeResult(code, null);
            codeResult.setFilePath(filepath);
            codeResult.setCodePath(doc.get(Values.FILELOCATIONFILENAME));
            codeResult.setFileName(doc.get(Values.FILENAME));
            codeResult.setLanguageName(doc.get(Values.LANGUAGENAME));
            codeResult.setMd5hash(doc.get(Values.MD5HASH));
            codeResult.setCodeLines(doc.get(Values.CODELINES));
            codeResult.setDocumentId(hits[0].doc);
            codeResult.setRepoName(doc.get(Values.REPONAME));
            codeResult.setRepoLocation(doc.get(Values.REPOLOCATION));
            codeResult.setCodeOwner(doc.get(Values.CODEOWNER));
            codeResult.setCodeId(doc.get(Values.CODEID));
        }

        reader.close();
    } catch (Exception ex) {
        LOGGER.severe(" caught a " + ex.getClass() + "\n with message: " + ex.getMessage());
    }

    return codeResult;
}

From source file:com.searchcode.app.service.CodeSearcher.java

License:Open Source License

public ProjectStats getProjectStats(String repoName) {
    int totalCodeLines = 0;
    int totalFiles = 0;
    List<CodeFacetLanguage> codeFacetLanguages = new ArrayList<>();
    List<CodeFacetOwner> repoFacetOwners = new ArrayList<>();
    List<CodeFacetLanguage> codeByLines = new ArrayList<>();
    SearchcodeLib searchcodeLib = Singleton.getSearchCodeLib();

    try {//from  w  w w .  j a  v a 2  s  .c o m
        IndexReader reader = DirectoryReader.open(FSDirectory.open(Paths.get(this.INDEXPATH)));
        IndexSearcher searcher = new IndexSearcher(reader);

        Analyzer analyzer = new CodeAnalyzer();
        QueryParser parser = new QueryParser(CODEFIELD, analyzer);
        Query query = parser.parse(Values.REPONAME + ":" + repoName);

        TopDocs results = searcher.search(query, Integer.MAX_VALUE);
        ScoreDoc[] hits = results.scoreDocs;

        Map<String, Integer> linesCount = new HashMap<>();

        for (int i = 0; i < results.totalHits; i++) {
            Document doc = searcher.doc(hits[i].doc);

            if (!searchcodeLib.languageCostIgnore(doc.get(Values.LANGUAGENAME))) {
                int lines = Singleton.getHelpers().tryParseInt(doc.get(Values.CODELINES), "0");
                totalCodeLines += lines;
                String languageName = doc.get(Values.LANGUAGENAME).replace("_", " ");

                if (linesCount.containsKey(languageName)) {
                    linesCount.put(languageName, linesCount.get(languageName) + lines);
                } else {
                    linesCount.put(languageName, lines);
                }
            }
        }

        for (String key : linesCount.keySet()) {
            codeByLines.add(new CodeFacetLanguage(key, linesCount.get(key)));
        }
        codeByLines.sort((a, b) -> b.getCount() - a.getCount());

        totalFiles = results.totalHits;
        codeFacetLanguages = this.getLanguageFacetResults(searcher, reader, query);
        repoFacetOwners = this.getOwnerFacetResults(searcher, reader, query);

        reader.close();
    } catch (Exception ex) {
        LOGGER.severe("CodeSearcher getProjectStats caught a " + ex.getClass() + "\n with message: "
                + ex.getMessage());
    }

    return new ProjectStats(totalCodeLines, totalFiles, codeFacetLanguages, codeByLines, repoFacetOwners);
}

From source file:com.searchcode.app.service.CodeSearcher.java

License:Open Source License

/**
 * Due to very large repositories (500,000 files) this needs to support
 * paging. Also need to consider the fact that is a list of strings
 * TODO maybe convert to hash so lookups are faster
 *//*from   www .  java  2  s .  co m*/
public List<String> getRepoDocuments(String repoName, int page) {
    int REPOPAGELIMIT = 1000;
    List<String> fileLocations = new ArrayList<>(REPOPAGELIMIT);
    int start = REPOPAGELIMIT * page;

    try {
        IndexReader reader = DirectoryReader.open(FSDirectory.open(Paths.get(this.INDEXPATH)));
        IndexSearcher searcher = new IndexSearcher(reader);

        Analyzer analyzer = new CodeAnalyzer();
        QueryParser parser = new QueryParser(CODEFIELD, analyzer);
        Query query = parser.parse(Values.REPONAME + ":" + repoName);

        TopDocs results = searcher.search(query, Integer.MAX_VALUE);
        int end = Math.min(results.totalHits, (REPOPAGELIMIT * (page + 1)));
        ScoreDoc[] hits = results.scoreDocs;

        for (int i = start; i < end; i++) {
            Document doc = searcher.doc(hits[i].doc);
            fileLocations.add(doc.get(Values.PATH));
        }

        reader.close();
    } catch (Exception ex) {
        LOGGER.severe("CodeSearcher getRepoDocuments caught a " + ex.getClass() + " on page " + page
                + "\n with message: " + ex.getMessage());
    }

    return fileLocations;
}

From source file:com.searchcode.app.service.IndexService.java

License:Open Source License

/**
 * Returns the total number of documents that are present in the index at this time
 */// ww w. j  a  v  a 2s .  c o m
public int getTotalNumberDocumentsIndexed() {
    int numDocs = 0;
    try {
        IndexReader reader = DirectoryReader.open(FSDirectory.open(this.INDEX_LOCATION));
        numDocs = reader.numDocs();
        reader.close();
    } catch (IOException ex) {
        this.logger.info(" caught a " + ex.getClass() + "\n with message: " + ex.getMessage());
    }

    return numDocs;
}

From source file:com.searchcode.app.service.TimeCodeSearcher.java

/**
 * Given a query and what page of results we are on return the matching results for that search
 *//* w w  w  .ja v a 2s .c  om*/
public SearchResult search(String queryString, int page) {
    SearchResult searchResult = new SearchResult();
    statsService.incrementSearchCount();

    try {
        IndexReader reader = DirectoryReader.open(FSDirectory.open(Paths.get(this.INDEXPATH)));
        IndexSearcher searcher = new IndexSearcher(reader);

        Analyzer analyzer = new CodeAnalyzer();

        QueryParser parser = new QueryParser(CODEFIELD, analyzer);

        Query query = parser.parse(queryString);
        LOGGER.info("Searching for: " + query.toString(CODEFIELD));

        searchResult = this.doPagingSearch(reader, searcher, query, page);
        reader.close();
    } catch (Exception ex) {
        LOGGER.warning(" caught a " + ex.getClass() + "\n with message: " + ex.getMessage());
    }

    return searchResult;
}

From source file:com.searchcode.app.service.TimeCodeSearcher.java

/**
 * Attempts to find a unique file given the repository name and the path/filename however
 * it seems to randomly not find things for some files. No idea of the root cause at this point and have implemented
 * a work around where we get the file by getById which is no ideal. The bug appears to be due to some issue
 * inside lucene itself as using raw queries to pull back the file results in no matches, and yet it does appear
 * when not limiting to the repo// w  ww . j av  a 2 s. c o m
 * TODO investigate the lucene issue that occurs here mentioned above
 * TODO needs to use the revision number here as well to get the right value
 */
public CodeResult getByRepoFileName(String repo, String fileName) {
    CodeResult codeResult = null;

    try {
        IndexReader reader = DirectoryReader.open(FSDirectory.open(Paths.get(this.INDEXPATH)));
        IndexSearcher searcher = new IndexSearcher(reader);
        Analyzer analyzer = new CodeAnalyzer();
        QueryParser parser = new QueryParser(CODEFIELD, analyzer);

        // TODO I have a feeling this may not be unique if there are to files in the same directory with different case... something to investigate
        Query query = parser
                .parse(Values.FILELOCATIONFILENAME + ":" + QueryParser.escape(repo + "/" + fileName));
        Singleton.getLogger().info("Query to get by filename = " + Values.FILELOCATIONFILENAME + ":"
                + QueryParser.escape(repo + "/" + fileName));

        TopDocs results = searcher.search(query, 1);
        ScoreDoc[] hits = results.scoreDocs;

        if (hits.length != 0) {
            Document doc = searcher.doc(hits[0].doc);

            String filepath = doc.get(Values.PATH);

            List<String> code = new ArrayList<>();
            try {
                code = Files.readAllLines(Paths.get(filepath), StandardCharsets.UTF_8);
                code = Singleton.getHelpers()
                        .readFileLines(filepath,
                                Singleton.getHelpers().tryParseInt(
                                        Properties.getProperties().getProperty(Values.MAXFILELINEDEPTH,
                                                Values.DEFAULTMAXFILELINEDEPTH),
                                        Values.DEFAULTMAXFILELINEDEPTH));
            } catch (Exception ex) {
                Singleton.getLogger().info("Indexed file appears to binary: " + filepath);
            }

            codeResult = new CodeResult(code, null);
            codeResult.setCodePath(doc.get(Values.FILELOCATIONFILENAME));
            codeResult.setFileName(doc.get(Values.FILENAME));
            codeResult.setLanguageName(doc.get(Values.LANGUAGENAME));
            codeResult.setMd5hash(doc.get(Values.MD5HASH));
            codeResult.setCodeLines(doc.get(Values.CODELINES));
            codeResult.setDocumentId(hits[0].doc);
            codeResult.setRepoName(doc.get(Values.REPONAME));
            codeResult.setRepoLocation(doc.get(Values.REPOLOCATION));
            codeResult.setCodeOwner(doc.get(Values.CODEOWNER));
        }

        reader.close();

    } catch (Exception ex) {
        LOGGER.severe(" caught a " + ex.getClass() + "\n with message: " + ex.getMessage());
    }

    return codeResult;
}

From source file:com.searchcode.app.service.TimeCodeSearcher.java

/**
 * Only used as fallback if getByRepoFileName fails for some reason due to what appears to be a lucene index bug
 * Using this is problematic because if the index is updated while this method is called it will possibly
 * return the incorrect result. We could add a shared lock between them both but that's hardly ideal especially
 * since when its called the index could already be updated
 *///w w w  .  j  a  v a 2s  .  c o m
public CodeResult getById(int documentId) {
    CodeResult codeResult = null;

    try {
        IndexReader reader = DirectoryReader.open(FSDirectory.open(Paths.get(this.INDEXPATH)));
        Document doc = reader.document(documentId);

        String filepath = doc.get(Values.PATH);

        List<String> code = new ArrayList<>();
        try {
            code = Files.readAllLines(Paths.get(filepath), StandardCharsets.UTF_8);
        } catch (Exception ex) {
            LOGGER.warning("Indexed file appears to binary: " + filepath);
        }

        codeResult = new CodeResult(code, null);
        codeResult.setCodePath(doc.get(Values.FILELOCATIONFILENAME));
        codeResult.setFileName(doc.get(Values.FILENAME));
        codeResult.setLanguageName(doc.get(Values.LANGUAGENAME));
        codeResult.setMd5hash(doc.get(Values.MD5HASH));
        codeResult.setCodeLines(doc.get(Values.CODELINES));
        codeResult.setDocumentId(documentId);
        codeResult.setRepoName(doc.get(Values.REPONAME));
        codeResult.setRepoLocation(doc.get(Values.REPOLOCATION));
        codeResult.setCodeOwner(doc.get(Values.CODEOWNER));

        reader.close();
    } catch (Exception ex) {
        LOGGER.warning(" caught a " + ex.getClass() + "\n with message: " + ex.getMessage());
    }

    return codeResult;
}

From source file:com.searchcode.app.service.TimeCodeSearcher.java

public List<String> getRepoDocuments(String repoName) {
    List<String> fileLocations = new ArrayList<>();
    try {/* www.  ja  va  2  s  .c o m*/
        IndexReader reader = DirectoryReader.open(FSDirectory.open(Paths.get(this.INDEXPATH)));
        IndexSearcher searcher = new IndexSearcher(reader);

        Analyzer analyzer = new CodeAnalyzer();
        QueryParser parser = new QueryParser(CODEFIELD, analyzer);
        Query query = parser.parse(Values.REPONAME + ":" + repoName);

        TopDocs results = searcher.search(query, Integer.MAX_VALUE);
        ScoreDoc[] hits = results.scoreDocs;

        for (int i = 0; i < hits.length; i++) {
            Document doc = searcher.doc(hits[i].doc);
            fileLocations.add(doc.get(Values.FILELOCATIONFILENAME));
        }

        reader.close();
    } catch (Exception ex) {
        LOGGER.severe(" caught a " + ex.getClass() + "\n with message: " + ex.getMessage());
    }

    return fileLocations;
}

From source file:com.sg.business.vault.index.demo.SearchFiles.java

License:Apache License

/** Simple command-line based search demo. */
public static void main(String[] args) throws Exception {
    String usage = "Usage:\tjava org.apache.lucene.demo.SearchFiles [-index dir] [-field f] [-repeat n] [-queries file] [-query string] [-raw] [-paging hitsPerPage]\n\nSee http://lucene.apache.org/core/4_1_0/demo/ for details."; //$NON-NLS-1$
    if (args.length > 0 && ("-h".equals(args[0]) || "-help".equals(args[0]))) { //$NON-NLS-1$ //$NON-NLS-2$
        System.out.println(usage);
        System.exit(0);/*from  w w w . j  av  a2  s  .c om*/
    }

    String index = "index"; //$NON-NLS-1$
    String field = "contents"; //$NON-NLS-1$
    String queries = null;
    int repeat = 0;
    boolean raw = false;
    String queryString = null;
    int hitsPerPage = 10;

    for (int i = 0; i < args.length; i++) {
        if ("-index".equals(args[i])) { //$NON-NLS-1$
            index = args[i + 1];
            i++;
        } else if ("-field".equals(args[i])) { //$NON-NLS-1$
            field = args[i + 1];
            i++;
        } else if ("-queries".equals(args[i])) { //$NON-NLS-1$
            queries = args[i + 1];
            i++;
        } else if ("-query".equals(args[i])) { //$NON-NLS-1$
            queryString = args[i + 1];
            i++;
        } else if ("-repeat".equals(args[i])) { //$NON-NLS-1$
            repeat = Integer.parseInt(args[i + 1]);
            i++;
        } else if ("-raw".equals(args[i])) { //$NON-NLS-1$
            raw = true;
        } else if ("-paging".equals(args[i])) { //$NON-NLS-1$
            hitsPerPage = Integer.parseInt(args[i + 1]);
            if (hitsPerPage <= 0) {
                System.err.println("There must be at least 1 hit per page."); //$NON-NLS-1$
                System.exit(1);
            }
            i++;
        }
    }

    IndexReader reader = DirectoryReader.open(FSDirectory.open(new File(index)));
    IndexSearcher searcher = new IndexSearcher(reader);
    Analyzer analyzer = new SmartChineseAnalyzer(Version.LUCENE_44);

    BufferedReader in = null;
    if (queries != null) {
        in = new BufferedReader(new InputStreamReader(new FileInputStream(queries), "UTF-8")); //$NON-NLS-1$
    } else {
        in = new BufferedReader(new InputStreamReader(System.in, "UTF-8")); //$NON-NLS-1$
    }
    QueryParser parser = new QueryParser(Version.LUCENE_44, field, analyzer);
    while (true) {
        if (queries == null && queryString == null) { // prompt the user
            System.out.println("Enter query: "); //$NON-NLS-1$
        }

        String line = queryString != null ? queryString : in.readLine();

        if (line == null || line.length() == -1) {
            break;
        }

        line = line.trim();
        if (line.length() == 0) {
            break;
        }

        Query query = parser.parse(line);
        System.out.println("Searching for: " + query.toString(field)); //$NON-NLS-1$

        if (repeat > 0) { // repeat & time as benchmark
            Date start = new Date();
            for (int i = 0; i < repeat; i++) {
                searcher.search(query, null, 100);
            }
            Date end = new Date();
            System.out.println("Time: " + (end.getTime() - start.getTime()) //$NON-NLS-1$
                    + "ms"); //$NON-NLS-1$
        }

        doPagingSearch(in, searcher, query, hitsPerPage, raw, queries == null && queryString == null);

        if (queryString != null) {
            break;
        }
    }
    reader.close();
}

From source file:com.silverpeas.silvercrawler.model.FileFolder.java

License:Open Source License

public FileFolder(String rootPath, String path, boolean isAdmin, String componentId) {
    files = new ArrayList<FileDetail>(0);
    folders = new ArrayList<FileDetail>(0);

    String childPath = null;//w  w w. j  a  v  a 2s.co m

    try {
        SilverTrace.debug("silverCrawler", "FileFolder.FileFolder()", "root.MSG_GEN_PARAM_VALUE",
                "Starting constructor for FileFolder. Path = " + path);
        File f = new File(path);
        File fChild;

        SilverTrace.debug("silverCrawler", "FileFolder.FileFolder()", "root.MSG_GEN_PARAM_VALUE",
                "isExists " + f.exists() + " isFile=" + f.isFile());

        writable = f.canWrite();

        if (f.exists()) {
            this.name = f.getName();

            this.readable = f.canRead();

            String[] children_name = f.list();

            IndexReader reader = null;
            boolean isIndexed = false;

            if (isAdmin) {
                // ouverture de l'index
                String indexPath = FileRepositoryManager.getAbsoluteIndexPath("", componentId);

                if (IndexReader.indexExists(indexPath))
                    reader = IndexReader.open(indexPath);
            }

            for (int i = 0; children_name != null && i < children_name.length; i++) {
                SilverTrace.debug("silverCrawler", "FileFolder.FileFolder()", "root.MSG_GEN_PARAM_VALUE",
                        "Name = " + children_name[i]);
                fChild = new File(path + File.separator + children_name[i]);
                isIndexed = false;
                if (isAdmin) {
                    // rechercher si le rpertoire (ou le fichier) est index
                    String pathIndex = componentId + "|";
                    if (fChild.isDirectory())
                        pathIndex = pathIndex + "LinkedDir" + "|";
                    else
                        pathIndex = pathIndex + "LinkedFile" + "|";
                    pathIndex = pathIndex + fChild.getPath();
                    SilverTrace.debug("silverCrawler", "FileFolder.FileFolder()", "root.MSG_GEN_PARAM_VALUE",
                            "pathIndex = " + pathIndex);

                    Term term = new Term("key", pathIndex);
                    if (reader != null && reader.docFreq(term) == 1)
                        isIndexed = true;
                }

                if (fChild.isDirectory()) {
                    folders.add(new FileDetail(fChild.getName(), fChild.getPath(), fChild.length(), true,
                            isIndexed));
                } else {
                    childPath = fChild.getPath().substring(rootPath.length() + 1);
                    files.add(new FileDetail(fChild.getName(), childPath, fChild.length(), false, isIndexed));
                }
            }

            // fermeture de l'index
            if (reader != null && isAdmin)
                reader.close();

        }
    } catch (Exception e) {
        throw new SilverCrawlerRuntimeException("FileFolder.FileFolder()", SilverpeasRuntimeException.ERROR,
                "silverCrawler.IMPOSSIBLE_DACCEDER_AU_REPERTOIRE", e);
    }
}