Example usage for org.apache.commons.httpclient URI setFragment

Introduction

In this page you can find the example usage for org.apache.commons.httpclient URI setFragment.

Prototype

public void setFragment(String fragment) throws URIException 

Source Link

Document

Set the fragment.

Usage

From source file:de.fuberlin.wiwiss.marbles.loading.DereferencerBatch.java

/**
 * Loads URL if not yet loaded/* w  ww. ja va  2  s  .  co m*/
 * 
 * @param url   The URL to load
 * @param step   The distance from the focal resource
 * @param redirectCount   The number of redirects performed in the course of this individual request
 * @param forceReload   Set this to true if the URL should be loaded even if a valid copy is already in the cache
 * @throws URIException
 */
public void loadURL(URI url, int step, int redirectCount, boolean forceReload) throws URIException {
    if (step > maxSteps || redirectCount > maxRedirects)
        return;

    /* Cut off local names from URI */
    url.setFragment("");

    if (retrievedURLs.contains(
            url)) /* force reload doesn't apply on batch level, as they are short-lived and this could cause infinite loops */
        return;

    if (!forceReload && cacheController.hasURLData(url.toString())) {
        /* Treat as retrieved when reading from cache */
        retrievedURLs.add(url);

        String redirect = cacheController.getCachedRedirect(url.toString());

        /* Process a cached redirect */
        if (redirect != null) {
            URI redirectUrl = new URI(url, redirect, true);
            loadURL(redirectUrl, step, redirectCount + 1, forceReload);
        } else {
            /* Data is already loaded; try to find new links within it */
            try {
                org.openrdf.model.URI sesameUri = new URIImpl(url.toString());
                processLinks(step + 1, sesameUri);
            } catch (IllegalArgumentException e) {
                e.printStackTrace();
            }
        }
    } else {
        /* No data about this URL; get it */
        ExtendedDereferencingTask task = new ExtendedDereferencingTask(this, url.toString(), step,
                redirectCount, forceReload);
        if (uriQueue.addTask(task)) {
            pendingTasks.add(task);
            retrievedURLs.add(url);
        }
    }
}

---