Java tutorial
/* * Copyright 2016 The Simple File Server Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.sfs.elasticsearch.account; import com.google.common.base.Optional; import io.vertx.core.MultiMap; import io.vertx.core.logging.Logger; import org.elasticsearch.action.search.ClearScrollRequestBuilder; import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.search.SearchScrollRequestBuilder; import org.elasticsearch.search.SearchHits; import org.sfs.Server; import org.sfs.SfsRequest; import org.sfs.VertxContext; import org.sfs.auth.AuthProviderService; import org.sfs.elasticsearch.Elasticsearch; import org.sfs.elasticsearch.Jsonify; import org.sfs.elasticsearch.container.LoadContainerStats; import org.sfs.rx.ToVoid; import org.sfs.vo.ContainerList; import org.sfs.vo.ObjectPath; import org.sfs.vo.PersistentAccount; import org.sfs.vo.PersistentContainer; import rx.Observable; import rx.functions.Func1; import java.util.NavigableMap; import java.util.TreeMap; import static com.google.common.base.Strings.isNullOrEmpty; import static com.google.common.primitives.Ints.tryParse; import static io.vertx.core.logging.LoggerFactory.getLogger; import static java.lang.Integer.valueOf; import static java.lang.String.format; import static org.elasticsearch.common.unit.TimeValue.timeValueMillis; import static org.elasticsearch.index.query.QueryBuilders.prefixQuery; import static org.elasticsearch.search.sort.SortOrder.ASC; import static org.elasticsearch.search.sort.SortParseElement.DOC_FIELD_NAME; import static org.sfs.rx.Defer.just; import static org.sfs.util.SfsHttpQueryParams.DELIMITER; import static org.sfs.util.SfsHttpQueryParams.END_MARKER; import static org.sfs.util.SfsHttpQueryParams.LIMIT; import static org.sfs.util.SfsHttpQueryParams.MARKER; import static org.sfs.util.SfsHttpQueryParams.PREFIX; import static org.sfs.util.UrlScaper.unescape; import static org.sfs.vo.ContainerList.SparseContainer; import static org.sfs.vo.ObjectPath.DELIMITER_LENGTH; import static org.sfs.vo.PersistentContainer.fromSearchHit; import static rx.Observable.empty; import static rx.Observable.from; public class ListContainers implements Func1<PersistentAccount, Observable<ContainerList>> { private static final Logger LOGGER = getLogger(ListContainers.class); private final SfsRequest sfsRequest; private final VertxContext<Server> vertxContext; private AuthProviderService authProviderService; public ListContainers(SfsRequest sfsRequest) { this.sfsRequest = sfsRequest; this.vertxContext = sfsRequest.vertxContext(); this.authProviderService = vertxContext.verticle().authProviderService(); } @Override public Observable<ContainerList> call(PersistentAccount account) { MultiMap queryParams = sfsRequest.params(); Elasticsearch elasticSearch = vertxContext.verticle().elasticsearch(); final String limit = queryParams.get(LIMIT); String marker = unescape(queryParams.get(MARKER)); String endMarker = unescape(queryParams.get(END_MARKER)); final String prefix = unescape(queryParams.get(PREFIX)); final String delimiter = unescape(queryParams.get(DELIMITER)); Integer parsedLimit = !isNullOrEmpty(limit) ? tryParse(limit) : valueOf(10000); parsedLimit = parsedLimit == null || parsedLimit < 0 || parsedLimit > 10000 ? 10000 : parsedLimit; String accountId = account.getId(); String accountPrefix = accountId + ObjectPath.DELIMITER; if (!isNullOrEmpty(prefix)) { accountPrefix += prefix; } final SearchRequestBuilder scrollRequest = elasticSearch.get().prepareSearch(elasticSearch.containerIndex()) .setTypes(elasticSearch.defaultType()).addSort(DOC_FIELD_NAME, ASC) .setScroll(timeValueMillis(elasticSearch.getDefaultScrollTimeout())) .setTimeout(timeValueMillis(elasticSearch.getDefaultSearchTimeout() - 10)) .setQuery(prefixQuery("_id", accountPrefix)).setSize(100); final Integer finalParsedLimit = parsedLimit; final NavigableMap<String, SparseContainer> listedObjects = new TreeMap<>(); return scan(account, prefix, delimiter, marker, endMarker, finalParsedLimit, elasticSearch, scrollRequest, listedObjects).map(aVoid -> new ContainerList(account, listedObjects.values())); } protected Observable<Void> scan(final PersistentAccount account, final String prefix, final String delimiter, final String marker, final String endMarker, final int limit, final Elasticsearch elasticsearch, final SearchRequestBuilder scrollRequest, final NavigableMap<String, SparseContainer> listedContainers) { if (LOGGER.isDebugEnabled()) { LOGGER.debug(format("Search Request = %s", Jsonify.toString(scrollRequest))); } return elasticsearch.execute(vertxContext, scrollRequest, elasticsearch.getDefaultSearchTimeout()) .map(Optional::get).flatMap(searchResponse -> { if (LOGGER.isDebugEnabled()) { LOGGER.debug(format("Search Response = %s", Jsonify.toString(searchResponse))); } SearchHits hits = searchResponse.getHits(); return toIterable(account, prefix, delimiter, marker, endMarker, hits) .doOnNext(sparseContainer -> { String id = sparseContainer.getContainerName(); SparseContainer existing = listedContainers.get(id); if (existing == null) { listedContainers.put(id, sparseContainer); } else { existing.setByteCount(existing.getByteCount() + sparseContainer.getByteCount()); existing.setObjectCount( existing.getObjectCount() + sparseContainer.getObjectCount()); } if (listedContainers.size() > limit) { listedContainers.pollLastEntry(); } }).count().map(new ToVoid<>()) .flatMap(aVoid -> scroll(account, prefix, delimiter, marker, endMarker, limit, elasticsearch, searchResponse.getScrollId(), listedContainers)); }); } protected Observable<Void> scroll(final PersistentAccount account, final String prefix, final String delimiter, final String marker, final String endMarker, final int limit, final Elasticsearch elasticsearch, final String scrollId, final NavigableMap<String, SparseContainer> listedContainers) { SearchScrollRequestBuilder scrollRequest = elasticsearch.get().prepareSearchScroll(scrollId) .setScroll(timeValueMillis(elasticsearch.getDefaultScrollTimeout())); if (LOGGER.isDebugEnabled()) { LOGGER.debug(format("Search Request = %s", Jsonify.toString(scrollRequest))); } return elasticsearch.execute(vertxContext, scrollRequest, elasticsearch.getDefaultSearchTimeout()) .map(Optional::get).flatMap(searchResponse -> { if (LOGGER.isDebugEnabled()) { LOGGER.debug(format("Search Response = %s", Jsonify.toString(searchResponse))); } SearchHits hits = searchResponse.getHits(); int numberOfHits = hits.getHits().length; if (numberOfHits > 0) { return toIterable(account, prefix, delimiter, marker, endMarker, hits) .doOnNext(sparseContainer -> { String id = sparseContainer.getContainerName(); SparseContainer existing = listedContainers.get(id); if (existing == null) { listedContainers.put(id, sparseContainer); } else { existing.setByteCount( existing.getByteCount() + sparseContainer.getByteCount()); existing.setObjectCount( existing.getObjectCount() + sparseContainer.getObjectCount()); } if (listedContainers.size() > limit) { listedContainers.pollLastEntry(); } }).count().map(new ToVoid<>()) .flatMap(aVoid -> scroll(account, prefix, delimiter, marker, endMarker, limit, elasticsearch, searchResponse.getScrollId(), listedContainers)); } else { return clearScroll(elasticsearch, searchResponse.getScrollId()); } }); } protected Observable<Void> clearScroll(Elasticsearch elasticSearch, String scrollId) { ClearScrollRequestBuilder request = elasticSearch.get().prepareClearScroll().addScrollId(scrollId); return elasticSearch.execute(vertxContext, request, elasticSearch.getDefaultSearchTimeout()) .onErrorResumeNext(throwable -> { LOGGER.warn("Handling Clear Scroll Error", throwable); return just(null); }).map(clearScrollResponseOptional -> null); } protected Observable<SparseContainer> toIterable(final PersistentAccount account, final String prefix, final String delimiter, final String marker, final String endMarker, SearchHits searchHits) { // container id looks like /account/container // object id looks like /account/container/a/b/c/1/2/3 // which makes the start index of the object name is the length of the // container id + 1 int containerNameStartIndex = account.getId().length() + DELIMITER_LENGTH; return from(searchHits).flatMap(searchHit -> { String containerId = searchHit.getId(); containerId = containerId.substring(containerNameStartIndex, containerId.length()); if (delimiter != null) { int prefixLength = prefix != null ? prefix.length() : 0; int containerIdLength = containerId.length(); if (containerIdLength > prefixLength) { int indexOfDelimiter = containerId.indexOf(delimiter, prefixLength); if (indexOfDelimiter <= containerIdLength && indexOfDelimiter >= 0) { containerId = containerId.substring(0, indexOfDelimiter); } } } boolean isInRange = (marker == null || containerId.compareTo(marker) > 0) && (endMarker == null || containerId.compareTo(endMarker) < 0); if (isInRange) { PersistentContainer persistentContainer = fromSearchHit(account, searchHit); String finalContainerId = containerId; return authProviderService.canContainerRead(sfsRequest, persistentContainer).filter(canDo -> canDo) .map(canDo -> persistentContainer).flatMap(new LoadContainerStats(vertxContext)) .map(containerStats -> { long objectCount = containerStats.getObjectCount(); double bytesUsed = containerStats.getBytesUsed(); SparseContainer sparseContainer = new SparseContainer(finalContainerId, objectCount, bytesUsed); return sparseContainer; }); } return empty(); }); } }