org.locationtech.geogig.osm.internal.history.HistoryDownloader.java Source code

Introduction

Here is the source code for org.locationtech.geogig.osm.internal.history.HistoryDownloader.java

Source

/* Copyright (c) 2013-2014 Boundless and others.
 * All rights reserved. This program and the accompanying materials
 * are made available under the terms of the Eclipse Distribution License v1.0
 * which accompanies this distribution, and is available at
 * https://www.eclipse.org/org/documents/edl-v10.html
 *
 * Contributors:
 * Victor Olaya (Boundless) - initial implementation
 */
package org.locationtech.geogig.osm.internal.history;

import static com.google.common.base.Preconditions.checkArgument;

import java.io.BufferedInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.util.Iterator;
import java.util.List;
import java.util.concurrent.ExecutorService;

import javax.xml.stream.XMLStreamException;

import com.google.common.base.Function;
import com.google.common.base.Optional;
import com.google.common.base.Predicate;
import com.google.common.base.Predicates;
import com.google.common.base.Supplier;
import com.google.common.base.Throwables;
import com.google.common.collect.AbstractIterator;
import com.google.common.collect.ContiguousSet;
import com.google.common.collect.DiscreteDomain;
import com.google.common.collect.Iterators;
import com.google.common.collect.Range;
import com.google.common.io.Closeables;

/**
 *
 */
public class HistoryDownloader {

    private final long initialChangeset;

    private final long finalChangeset;

    private final ChangesetDownloader downloader;

    private Predicate<Changeset> filter = Predicates.alwaysTrue();

    /**
     * @param osmAPIUrl api url, e.g. {@code http://api.openstreetmap.org/api/0.6},
     *        {@code file:/path/to/downloaded/changesets}
     * @param initialChangeset initial changeset id
     * @param finalChangeset final changeset id
     * @param preserveFiles
     */
    public HistoryDownloader(final String osmAPIUrl, final File downloadFolder, long initialChangeset,
            long finalChangeset, ExecutorService executor) {

        checkArgument(initialChangeset > 0 && initialChangeset <= finalChangeset);

        this.initialChangeset = initialChangeset;
        this.finalChangeset = finalChangeset;
        this.downloader = new ChangesetDownloader(osmAPIUrl, downloadFolder, executor);
    }

    public void setChangesetFilter(Predicate<Changeset> filter) {
        this.filter = filter;
    }

    /**
    *
    */
    private class ChangesSupplier implements Supplier<Optional<Iterator<Change>>> {

        private Supplier<Optional<File>> changesFile;

        /**
         * @param changesFile2
         */
        public ChangesSupplier(Supplier<Optional<File>> changesFile) {
            this.changesFile = changesFile;
        }

        @Override
        public Optional<Iterator<Change>> get() {
            return parseChanges(changesFile);
        }

    }

    /**
     * @return the next available changeset, or absent if reached the last one
     * @throws IOException
     * @throws InterruptedException
     */
    public Iterator<Changeset> fetchChangesets() {

        Range<Long> range = Range.closed(initialChangeset, finalChangeset);
        ContiguousSet<Long> changesetIds = ContiguousSet.create(range, DiscreteDomain.longs());
        final int fetchSize = 100;
        Iterator<List<Long>> partitions = Iterators.partition(changesetIds.iterator(), fetchSize);

        Function<List<Long>, Iterator<Changeset>> asChangesets = new Function<List<Long>, Iterator<Changeset>>() {
            @Override
            public Iterator<Changeset> apply(List<Long> batchIds) {

                Iterable<Changeset> changesets = downloader.fetchChangesets(batchIds);

                for (Changeset changeset : changesets) {
                    if (filter.apply(changeset)) {
                        Supplier<Optional<File>> changesFile;
                        changesFile = downloader.fetchChanges(changeset.getId());
                        Supplier<Optional<Iterator<Change>>> changes = new ChangesSupplier(changesFile);
                        changeset.setChanges(changes);
                    }
                }

                return changesets.iterator();
            }
        };

        Iterator<Iterator<Changeset>> changesets = Iterators.transform(partitions, asChangesets);
        Iterator<Changeset> concat = Iterators.concat(changesets);
        return concat;
    }

    private Optional<Iterator<Change>> parseChanges(Supplier<Optional<File>> file) {

        final Optional<File> changesFile;
        try {
            changesFile = file.get();
        } catch (RuntimeException e) {
            Throwable cause = e.getCause();
            if (cause instanceof FileNotFoundException) {
                return Optional.absent();
            }
            throw Throwables.propagate(e);
        }
        if (!changesFile.isPresent()) {
            return Optional.absent();
        }
        final File actualFile = changesFile.get();
        final InputStream stream = openStream(actualFile);
        final Iterator<Change> changes;
        ChangesetContentsScanner scanner = new ChangesetContentsScanner();
        try {
            changes = scanner.parse(stream);
        } catch (XMLStreamException e) {
            throw Throwables.propagate(e);
        }

        Iterator<Change> iterator = new AbstractIterator<Change>() {
            @Override
            protected Change computeNext() {
                if (!changes.hasNext()) {
                    Closeables.closeQuietly(stream);
                    actualFile.delete();
                    actualFile.getParentFile().delete();
                    return super.endOfData();
                }
                return changes.next();
            }
        };
        return Optional.of(iterator);
    }

    private InputStream openStream(File file) {
        InputStream stream;
        try {
            stream = new BufferedInputStream(new FileInputStream(file), 4096);
        } catch (FileNotFoundException e) {
            throw Throwables.propagate(e);
        }
        return stream;
    }

}