com.thinkbiganalytics.feedmgr.rest.controller.FeedRestController.java Source code

Java tutorial

Introduction

Here is the source code for com.thinkbiganalytics.feedmgr.rest.controller.FeedRestController.java

Source

package com.thinkbiganalytics.feedmgr.rest.controller;

/*-
 * #%L
 * thinkbig-feed-manager-controller
 * %%
 * Copyright (C) 2017 ThinkBig Analytics
 * %%
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 * 
 *     http://www.apache.org/licenses/LICENSE-2.0
 * 
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 * #L%
 */

import com.google.common.collect.Lists;
import com.mifmif.common.regex.Generex;
import com.thinkbiganalytics.annotations.AnnotatedFieldProperty;
import com.thinkbiganalytics.annotations.AnnotationFieldNameResolver;
import com.thinkbiganalytics.discovery.schema.QueryResult;
import com.thinkbiganalytics.feedmgr.rest.model.EditFeedEntity;
import com.thinkbiganalytics.feedmgr.rest.model.FeedMetadata;
import com.thinkbiganalytics.feedmgr.rest.model.FeedSummary;
import com.thinkbiganalytics.feedmgr.rest.model.FeedVersions;
import com.thinkbiganalytics.feedmgr.rest.model.NifiFeed;
import com.thinkbiganalytics.feedmgr.rest.model.UIFeed;
import com.thinkbiganalytics.feedmgr.security.FeedServicesAccessControl;
import com.thinkbiganalytics.feedmgr.service.FeedCleanupFailedException;
import com.thinkbiganalytics.feedmgr.service.FeedCleanupTimeoutException;
import com.thinkbiganalytics.feedmgr.service.MetadataService;
import com.thinkbiganalytics.feedmgr.service.datasource.DatasourceService;
import com.thinkbiganalytics.feedmgr.service.feed.DuplicateFeedNameException;
import com.thinkbiganalytics.feedmgr.service.feed.FeedManagerPreconditionService;
import com.thinkbiganalytics.feedmgr.service.feed.FeedModelTransform;
import com.thinkbiganalytics.feedmgr.service.security.SecurityService;
import com.thinkbiganalytics.feedmgr.service.template.RegisteredTemplateService;
import com.thinkbiganalytics.feedmgr.sla.ServiceLevelAgreementService;
import com.thinkbiganalytics.hive.service.HiveService;
import com.thinkbiganalytics.hive.util.HiveUtils;
import com.thinkbiganalytics.metadata.FeedPropertySection;
import com.thinkbiganalytics.metadata.FeedPropertyType;
import com.thinkbiganalytics.metadata.modeshape.versioning.VersionNotFoundException;
import com.thinkbiganalytics.metadata.api.security.MetadataAccessControl;
import com.thinkbiganalytics.metadata.rest.model.data.DatasourceDefinition;
import com.thinkbiganalytics.metadata.rest.model.data.DatasourceDefinitions;
import com.thinkbiganalytics.metadata.rest.model.feed.FeedLineageStyle;
import com.thinkbiganalytics.metadata.rest.model.sla.FeedServiceLevelAgreement;
import com.thinkbiganalytics.nifi.rest.client.NifiClientRuntimeException;
import com.thinkbiganalytics.nifi.rest.model.NifiProperty;
import com.thinkbiganalytics.nifi.rest.support.NifiPropertyUtil;
import com.thinkbiganalytics.policy.rest.model.PreconditionRule;
import com.thinkbiganalytics.rest.model.RestResponseStatus;
import com.thinkbiganalytics.rest.model.search.SearchResult;
import com.thinkbiganalytics.security.AccessController;
import com.thinkbiganalytics.security.rest.controller.SecurityModelTransform;
import com.thinkbiganalytics.security.rest.model.ActionGroup;
import com.thinkbiganalytics.security.rest.model.PermissionsChange;
import com.thinkbiganalytics.security.rest.model.PermissionsChange.ChangeType;
import com.thinkbiganalytics.security.rest.model.RoleMembershipChange;
import com.thinkbiganalytics.security.rest.model.RoleMemberships;
import com.thinkbiganalytics.support.FeedNameUtil;

import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.reflect.FieldUtils;
import org.apache.directory.api.util.Strings;
import org.glassfish.jersey.media.multipart.FormDataContentDisposition;
import org.glassfish.jersey.media.multipart.FormDataParam;
import org.hibernate.JDBCException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.dao.DataAccessException;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.PageRequest;
import org.springframework.data.domain.Sort;
import org.springframework.stereotype.Component;

import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.AccessDeniedException;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.nio.file.StandardCopyOption;
import java.security.AccessControlException;
import java.security.Principal;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.ResourceBundle;
import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.Stream;

import javax.annotation.Nonnull;
import javax.inject.Inject;
import javax.ws.rs.ClientErrorException;
import javax.ws.rs.Consumes;
import javax.ws.rs.DELETE;
import javax.ws.rs.DefaultValue;
import javax.ws.rs.GET;
import javax.ws.rs.InternalServerErrorException;
import javax.ws.rs.NotFoundException;
import javax.ws.rs.POST;
import javax.ws.rs.Path;
import javax.ws.rs.PathParam;
import javax.ws.rs.Produces;
import javax.ws.rs.QueryParam;
import javax.ws.rs.WebApplicationException;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
import javax.ws.rs.core.Response.Status;

import io.swagger.annotations.Api;
import io.swagger.annotations.ApiOperation;
import io.swagger.annotations.ApiResponse;
import io.swagger.annotations.ApiResponses;
import io.swagger.annotations.SwaggerDefinition;
import io.swagger.annotations.Tag;

@Api(tags = "Feed Manager - Feeds", produces = "application/json")
@Path(FeedRestController.BASE)
@Component
@SwaggerDefinition(tags = @Tag(name = "Feed Manager - Feeds", description = "manages feeds"))
public class FeedRestController {

    private static final Logger log = LoggerFactory.getLogger(FeedRestController.class);
    public static final String BASE = "/v1/feedmgr/feeds";

    /**
     * Messages for the default locale
     */
    private static final ResourceBundle STRINGS = ResourceBundle
            .getBundle("com.thinkbiganalytics.feedmgr.rest.controller.FeedMessages");
    private static final int MAX_LIMIT = 1000;
    private static final String NAMES = "/names";
    private static final String SUMMARY = "/feed-summary";

    @Inject
    private MetadataService metadataService;

    @Inject
    private HiveService hiveService;

    @Inject
    private FeedManagerPreconditionService feedManagerPreconditionService;

    @Inject
    private FeedModelTransform feedModelTransform;

    @Inject
    private DatasourceService datasourceService;

    @Inject
    private SecurityService securityService;

    @Inject
    private SecurityModelTransform securityTransform;

    @Inject
    private ServiceLevelAgreementService serviceLevelAgreementService;

    @Inject
    private RegisteredTemplateService registeredTemplateService;

    @Inject
    private AccessController accessController;

    private MetadataService getMetadataService() {
        return metadataService;
    }

    /**
     * Creates a new Feed using the specified metadata.
     *
     * @param editFeedEntity the feed metadata
     * @return the feed
     */
    @POST
    @Path("/edit/{feedId}")
    @Consumes({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_FORM_URLENCODED })
    @Produces(MediaType.APPLICATION_JSON)
    @ApiOperation("Creates or updates a feed.")
    @ApiResponses(@ApiResponse(code = 200, message = "Returns the feed including any error messages.", response = NifiFeed.class))
    @Nonnull
    public Response editFeed(@Nonnull final EditFeedEntity editFeedEntity) {

        return createFeed(editFeedEntity.getFeedMetadata());
    }

    private void populateFeed(EditFeedEntity editFeedEntity) {
        //fetch the feed
        FeedMetadata feed = getMetadataService().getFeedById(editFeedEntity.getFeedMetadata().getFeedId());
        FeedMetadata editFeed = editFeedEntity.getFeedMetadata();
        switch (editFeedEntity.getAction()) {
        case SUMMARY:
            updateFeedMetadata(feed, editFeed, FeedPropertySection.SUMMARY);
            break;
        case NIFI_PROPERTIES:
            updateFeedMetadata(feed, editFeed, FeedPropertySection.NIFI_PROPERTIES);
            break;
        case PROPERTIES:
            updateFeedMetadata(feed, editFeed, FeedPropertySection.PROPERTIES);
            break;
        case TABLE_DATA:
            updateFeedMetadata(feed, editFeed, FeedPropertySection.TABLE_DATA);
            break;
        case SCHEDULE:
            updateFeedMetadata(feed, editFeed, FeedPropertySection.SCHEDULE);
            break;
        default:
            break;
        }

        createFeed(feed);
    }

    private void updateFeedMetadata(FeedMetadata targetFeedMetadata, FeedMetadata modifiedFeedMetadata,
            FeedPropertySection feedPropertySection) {

        AnnotationFieldNameResolver annotationFieldNameResolver = new AnnotationFieldNameResolver(
                FeedPropertyType.class);
        List<AnnotatedFieldProperty> list = annotationFieldNameResolver.getProperties(FeedMetadata.class);
        List<AnnotatedFieldProperty> sectionList = list.stream()
                .filter(annotatedFieldProperty -> feedPropertySection
                        .equals(((FeedPropertyType) annotatedFieldProperty.getAnnotation()).section()))
                .collect(Collectors.toList());
        sectionList.forEach(annotatedFieldProperty -> {
            try {
                Object value = FieldUtils.readField(annotatedFieldProperty.getField(), modifiedFeedMetadata);
                FieldUtils.writeField(annotatedFieldProperty.getField(), targetFeedMetadata, value);
            } catch (IllegalAccessException e) {
                e.printStackTrace();
            }
        });

    }

    /**
     * Creates a new Feed using the specified metadata.
     *
     * @param feedMetadata the feed metadata
     * @return the feed
     */
    @POST
    @Consumes({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_FORM_URLENCODED })
    @Produces(MediaType.APPLICATION_JSON)
    @ApiOperation("Creates or updates a feed.")
    @ApiResponses(@ApiResponse(code = 200, message = "Returns the feed including any error messages.", response = NifiFeed.class))
    @Nonnull
    public Response createFeed(@Nonnull final FeedMetadata feedMetadata) {
        NifiFeed feed;
        try {
            feed = getMetadataService().createFeed(feedMetadata);
        } catch (DuplicateFeedNameException e) {
            log.info("Failed to create a new feed due to another feed having the same category/feed name: "
                    + feedMetadata.getCategoryAndFeedDisplayName());

            // Create an error message
            String msg = "A feed already exists in the category \"" + e.getCategoryName() + "\" with name name \""
                    + e.getFeedName() + "\"";

            // Add error message to feed
            feed = new NifiFeed(feedMetadata, null);
            feed.addErrorMessage(msg);
            feed.setSuccess(false);
        } catch (Exception e) {
            log.error("Failed to create a new feed.", e);

            // Create an error message
            String msg = (e.getMessage() != null) ? "Error saving Feed " + e.getMessage()
                    : "An unknown error occurred while saving the feed.";
            if (e.getCause() instanceof JDBCException) {
                msg += ". " + ((JDBCException) e).getSQLException();
            }

            // Add error message to feed
            feed = new NifiFeed(feedMetadata, null);
            feed.addErrorMessage(msg);
            feed.setSuccess(false);
        }
        return Response.ok(feed).build();
    }

    @POST
    @Path("/enable/{feedId}")
    @Consumes({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_FORM_URLENCODED })
    @Produces(MediaType.APPLICATION_JSON)
    @ApiOperation("Enables a feed.")
    @ApiResponses({ @ApiResponse(code = 200, message = "The feed was enabled.", response = FeedSummary.class),
            @ApiResponse(code = 500, message = "The feed could not be enabled.", response = RestResponseStatus.class) })
    public Response enableFeed(@PathParam("feedId") String feedId) {
        FeedSummary feed = getMetadataService().enableFeed(feedId);
        return Response.ok(feed).build();
    }

    @POST
    @Path("/disable/{feedId}")
    @Consumes({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_FORM_URLENCODED })
    @Produces(MediaType.APPLICATION_JSON)
    @ApiOperation("Disables a feed.")
    @ApiResponses({ @ApiResponse(code = 200, message = "The feed was disabled.", response = FeedSummary.class),
            @ApiResponse(code = 500, message = "The feed could not be disabled.", response = RestResponseStatus.class) })
    public Response disableFeed(@PathParam("feedId") String feedId) {
        FeedSummary feed = getMetadataService().disableFeed(feedId);
        return Response.ok(feed).build();
    }

    @GET
    @Deprecated
    @Path(NAMES)
    @Produces(MediaType.APPLICATION_JSON)
    @ApiOperation("Gets the list of feed summaries.")
    @ApiResponses(@ApiResponse(code = 200, message = "Returns a list of feeds.", response = FeedSummary.class, responseContainer = "List"))
    public Response getFeedNames() {
        Collection<FeedSummary> feeds = getMetadataService().getFeedSummaryData();
        return Response.ok(feeds).build();
    }

    @GET
    @Path(SUMMARY)
    @Produces(MediaType.APPLICATION_JSON)
    @ApiOperation("Gets the list of feed summaries.")
    @ApiResponses(@ApiResponse(code = 200, message = "Returns a list of feeds.", response = FeedSummary.class, responseContainer = "List"))
    public Response getFeedSummaries() {
        Collection<FeedSummary> feeds = getMetadataService().getFeedSummaryData();
        return Response.ok(feeds).build();
    }

    @GET
    @Produces(MediaType.APPLICATION_JSON)
    @ApiOperation("Gets the list of feeds.")
    @ApiResponses(@ApiResponse(code = 200, message = "Returns a list of feeds.", response = FeedMetadata.class, responseContainer = "List"))
    public SearchResult getFeeds(@QueryParam("verbose") @DefaultValue("false") boolean verbose,
            @QueryParam("sort") @DefaultValue("feedName") String sort, @QueryParam("filter") String filter,
            @QueryParam("limit") String limit, @QueryParam("start") @DefaultValue("0") Integer start) {

        try {
            int size = Strings.isEmpty(limit) || limit.equalsIgnoreCase("all") ? MAX_LIMIT
                    : Integer.parseInt(limit);
            Page<UIFeed> page = getMetadataService().getFeedsPage(verbose, pageRequest(start, size, sort),
                    filter != null ? filter.trim() : null);
            return this.feedModelTransform.toSearchResult(page);
        } catch (NumberFormatException e) {
            throw new IllegalArgumentException("The value of limit must be an integer or \"all\"");
        }
    }

    @GET
    @Path("/{feedId}")
    @Produces(MediaType.APPLICATION_JSON)
    @ApiOperation("Gets the specified feed.")
    @ApiResponses({ @ApiResponse(code = 200, message = "Returns the feed.", response = FeedMetadata.class),
            @ApiResponse(code = 500, message = "The feed is unavailable.", response = RestResponseStatus.class) })
    public Response getFeed(@PathParam("feedId") String feedId) {
        FeedMetadata feed = getMetadataService().getFeedById(feedId, true);

        return Response.ok(feed).build();
    }

    @GET
    @Path("/by-name/{feedName}")
    @Produces(MediaType.APPLICATION_JSON)
    @ApiOperation("Gets the specified feed.")
    @ApiResponses({ @ApiResponse(code = 200, message = "Returns the feed.", response = FeedMetadata.class),
            @ApiResponse(code = 500, message = "The feed is unavailable.", response = RestResponseStatus.class) })
    public Response getFeedByName(@PathParam("feedName") String feedName) {
        String categorySystemName = FeedNameUtil.category(feedName);
        String feedSystemName = FeedNameUtil.feed(feedName);
        if (StringUtils.isNotBlank(categorySystemName) && StringUtils.isNotBlank(feedSystemName)) {
            FeedMetadata feed = getMetadataService().getFeedByName(categorySystemName, feedSystemName);
            return Response.ok(feed).build();
        } else {
            throw new NotFoundException("Unable to find the feed for name: " + feedName);
        }
    }

    @GET
    @Path("/by-name/{feedName}/field-policies")
    @Produces(MediaType.APPLICATION_JSON)
    @ApiOperation("Gets the specified feed.")
    @ApiResponses({
            @ApiResponse(code = 200, message = "Returns the feed field policies (List<FieldPolicy>) as json.", response = List.class),
            @ApiResponse(code = 500, message = "The feed is unavailable.", response = RestResponseStatus.class) })
    public Response getFeedFieldPoliciesByName(@PathParam("feedName") String feedName) {
        String categorySystemName = FeedNameUtil.category(feedName);
        String feedSystemName = FeedNameUtil.feed(feedName);
        if (StringUtils.isNotBlank(categorySystemName) && StringUtils.isNotBlank(feedSystemName)) {
            FeedMetadata feed = getMetadataService().getFeedByName(categorySystemName, feedSystemName);
            if (feed != null && feed.getTable() != null) {
                return Response.ok(feed.getTable().getFieldPoliciesJson()).build();
            } else {
                throw new NotFoundException("Unable to find the feed field policies for name: " + feedName);
            }
        } else {
            throw new NotFoundException("Unable to find the feed field policies for name: " + feedName);
        }
    }

    /**
     * Deletes the specified feed.
     *
     * @param feedId the feed id
     * @return the response
     */
    @DELETE
    @Path("/{feedId}")
    @Produces(MediaType.APPLICATION_JSON)
    @ApiOperation("Deletes the specified feed.")
    @ApiResponses({ @ApiResponse(code = 204, message = "The feed was deleted."),
            @ApiResponse(code = 404, message = "The feed does not exist.", response = RestResponseStatus.class),
            @ApiResponse(code = 409, message = "There are dependent feeds.", response = RestResponseStatus.class),
            @ApiResponse(code = 500, message = "The feed could not be deleted.", response = RestResponseStatus.class) })
    @Nonnull
    public Response deleteFeed(@Nonnull @PathParam("feedId") final String feedId) {
        try {
            getMetadataService().deleteFeed(feedId);
            return Response.noContent().build();
        } catch (AccessControlException e) {
            log.debug("Access controll failure attempting to delete a feed", e);
            throw e;
        } catch (FeedCleanupFailedException e) {
            log.error("Error deleting feed: Cleanup error", e);
            throw new InternalServerErrorException(STRINGS.getString("deleteFeed.cleanupError"), e);
        } catch (FeedCleanupTimeoutException e) {
            log.error("Error deleting feed: Cleanup timeout", e);
            throw new InternalServerErrorException(STRINGS.getString("deleteFeed.cleanupTimeout"), e);
        } catch (IllegalArgumentException e) {
            log.error("Error deleting feed: Illegal Argument", e);
            throw new NotFoundException(STRINGS.getString("deleteFeed.notFound"), e);
        } catch (final IllegalStateException e) {
            log.error("Error deleting feed: Illegal state", e);
            throw new ClientErrorException(STRINGS.getString("deleteFeed.hasDependents"), Response.Status.CONFLICT,
                    e);
        } catch (NifiClientRuntimeException e) {
            log.error("Error deleting feed: NiFi error", e);
            throw new InternalServerErrorException(STRINGS.getString("deleteFeed.nifiError"), e);
        } catch (Exception e) {
            log.error("Error deleting feed: Unknown error", e);
            throw new InternalServerErrorException(STRINGS.getString("deleteFeed.unknownError"), e);
        }
    }

    @GET
    @Path("/{feedId}/versions")
    @Produces(MediaType.APPLICATION_JSON)
    @ApiOperation("Updates a feed with the latest template metadata.")
    @ApiResponses({ @ApiResponse(code = 200, message = "Returns the feed versions.", response = FeedMetadata.class),
            @ApiResponse(code = 500, message = "The feed is unavailable.", response = RestResponseStatus.class) })
    public Response getFeedVersions(@PathParam("feedId") String feedId,
            @QueryParam("content") @DefaultValue("true") boolean includeContent) {
        FeedVersions feed = getMetadataService().getFeedVersions(feedId, includeContent);

        return Response.ok(feed).build();
    }

    @GET
    @Path("/{feedId}/versions/{versionId}")
    @Produces(MediaType.APPLICATION_JSON)
    @ApiOperation("Updates a feed with the latest template metadata.")
    @ApiResponses({ @ApiResponse(code = 200, message = "Returns the feed versions.", response = FeedMetadata.class),
            @ApiResponse(code = 400, message = "Returns the feed or version does not exist.", response = FeedMetadata.class),
            @ApiResponse(code = 500, message = "The feed is unavailable.", response = RestResponseStatus.class) })
    public Response getFeedVersion(@PathParam("feedId") String feedId, @PathParam("versionId") String versionId,
            @QueryParam("content") @DefaultValue("true") boolean includeContent) {
        try {
            return getMetadataService().getFeedVersion(feedId, versionId, includeContent)
                    .map(version -> Response.ok(version).build()).orElse(Response.status(Status.NOT_FOUND).build());
        } catch (VersionNotFoundException e) {
            return Response.status(Status.NOT_FOUND).build();
        } catch (Exception e) {
            log.error("Unexpected exception retrieving the feed version", e);
            throw new InternalServerErrorException("Unexpected exception retrieving the feed version");
        }
    }

    @POST
    @Path("/{feedId}/merge-template")
    @Consumes({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_FORM_URLENCODED })
    @Produces(MediaType.APPLICATION_JSON)
    @ApiOperation("Updates a feed with the latest template metadata.")
    @ApiResponses({ @ApiResponse(code = 200, message = "The feed was updated.", response = FeedMetadata.class),
            @ApiResponse(code = 500, message = "The feed could not be updated.", response = RestResponseStatus.class) })
    public Response mergeTemplate(@PathParam("feedId") String feedId, FeedMetadata feed) {
        registeredTemplateService.mergeTemplatePropertiesWithFeed(feed);
        return Response.ok(feed).build();
    }

    @GET
    @Path("/{feedId}/profile-summary")
    @Produces(MediaType.APPLICATION_JSON)
    @ApiOperation("Gets a summary of the feed profiles.")
    @ApiResponses({
            @ApiResponse(code = 200, message = "Returns the profile summaries.", response = Map.class, responseContainer = "List"),
            @ApiResponse(code = 500, message = "The profiles are unavailable.", response = RestResponseStatus.class) })
    public Response profileSummary(@PathParam("feedId") String feedId) {
        FeedMetadata feedMetadata = getMetadataService().getFeedById(feedId);
        final String profileTable = HiveUtils.quoteIdentifier(feedMetadata.getProfileTableName());
        String query = "SELECT * from " + profileTable + " where columnname = '(ALL)'";

        List<Map<String, Object>> rows = new ArrayList<>();
        try {
            QueryResult results = hiveService.query(query);

            rows.addAll(results.getRows());
            //add in the archive date time fields if applicipable
            String ARCHIVE_PROCESSOR_TYPE = "com.thinkbiganalytics.nifi.GetTableData";
            if (feedMetadata.getInputProcessorType().equalsIgnoreCase(ARCHIVE_PROCESSOR_TYPE)) {
                NifiProperty property = NifiPropertyUtil.findPropertyByProcessorType(feedMetadata.getProperties(),
                        ARCHIVE_PROCESSOR_TYPE, "Date Field");
                if (property != null && property.getValue() != null) {
                    String field = property.getValue();
                    if (field.contains(".")) {
                        field = StringUtils.substringAfterLast(field, ".");
                    }
                    query = "SELECT * from " + profileTable
                            + " where metrictype IN('MIN_TIMESTAMP','MAX_TIMESTAMP') AND columnname = "
                            + HiveUtils.quoteString(field);

                    QueryResult dateRows = hiveService.query(query);
                    if (dateRows != null && !dateRows.isEmpty()) {
                        rows.addAll(dateRows.getRows());
                    }
                }
            }
        } catch (DataAccessException e) {
            if (e.getCause() instanceof org.apache.hive.service.cli.HiveSQLException
                    && e.getCause().getMessage().contains("Table not found")) {
                //this exception is ok to swallow since it just means no profile data exists yet
            } else if (e.getCause().getMessage().contains("HiveAccessControlException Permission denied")) {
                throw new AccessControlException("You do not have permission to execute this hive query");
            } else {
                throw e;
            }
        }

        return Response.ok(rows).build();
    }

    @GET
    @Path("/{feedId}/profile-stats")
    @Produces(MediaType.APPLICATION_JSON)
    @ApiOperation("Gets the profile statistics for the specified job.")
    @ApiResponses({
            @ApiResponse(code = 200, message = "Returns the profile statistics.", response = Map.class, responseContainer = "List"),
            @ApiResponse(code = 500, message = "The profile is unavailable.", response = RestResponseStatus.class) })
    public Response profileStats(@PathParam("feedId") String feedId,
            @QueryParam("processingdttm") String processingdttm) {
        FeedMetadata feedMetadata = getMetadataService().getFeedById(feedId);
        String profileTable = feedMetadata.getProfileTableName();
        String query = "SELECT * from " + HiveUtils.quoteIdentifier(profileTable) + " where processing_dttm = "
                + HiveUtils.quoteString(processingdttm);
        QueryResult rows = hiveService.query(query);
        return Response.ok(rows.getRows()).build();
    }

    @GET
    @Path("/{feedId}/profile-invalid-results")
    @Produces(MediaType.APPLICATION_JSON)
    @ApiOperation("Gets statistics on the invalid rows for the specified job.")
    @ApiResponses({
            @ApiResponse(code = 200, message = "Returns the invalid row statistics.", response = Map.class, responseContainer = "List"),
            @ApiResponse(code = 500, message = "The profile is unavailable.", response = RestResponseStatus.class) })
    public Response queryProfileInvalidResults(@PathParam("feedId") String feedId,
            @QueryParam("processingdttm") String processingdttm, @QueryParam("limit") int limit,
            @QueryParam("filter") String filter) {
        FeedMetadata feedMetadata = getMetadataService().getFeedById(feedId);
        String condition = "";
        if (StringUtils.isNotBlank(filter)) {
            condition = " and dlp_reject_reason like " + HiveUtils.quoteString("%" + filter + "%") + " ";
        }
        return getPage(processingdttm, limit, feedMetadata.getInvalidTableName(), condition);
    }

    @GET
    @Path("/{feedId}/profile-valid-results")
    @Produces(MediaType.APPLICATION_JSON)
    @ApiOperation("Gets statistics on the valid rows for the specified job.")
    @ApiResponses({
            @ApiResponse(code = 200, message = "Returns the valid row statistics.", response = Map.class, responseContainer = "List"),
            @ApiResponse(code = 500, message = "The profile is unavailable.", response = RestResponseStatus.class) })
    public Response queryProfileValidResults(@PathParam("feedId") String feedId,
            @QueryParam("processingdttm") String processingdttm, @QueryParam("limit") int limit) {
        FeedMetadata feedMetadata = getMetadataService().getFeedById(feedId);
        return getPage(processingdttm, limit, feedMetadata.getValidTableName());
    }

    @GET
    @Path("{feedId}/roles")
    @Produces(MediaType.APPLICATION_JSON)
    @ApiOperation("Gets the list of assigned members the feed's roles")
    @ApiResponses({
            @ApiResponse(code = 200, message = "Returns the role memberships.", response = ActionGroup.class),
            @ApiResponse(code = 404, message = "A feed with the given ID does not exist.", response = RestResponseStatus.class) })
    public Response getRoleMemberships(@PathParam("feedId") String feedIdStr,
            @QueryParam("verbose") @DefaultValue("false") boolean verbose) {
        // TODO: No longer using verbose; all results are verbose now.
        return this.securityService.getFeedRoleMemberships(feedIdStr).map(m -> Response.ok(m).build()).orElseThrow(
                () -> new WebApplicationException("A feed with the given ID does not exist: " + feedIdStr,
                        Status.NOT_FOUND));
    }

    @POST
    @Path("{feedId}/roles")
    @Consumes(MediaType.APPLICATION_JSON)
    @Produces(MediaType.APPLICATION_JSON)
    @ApiOperation("Updates the members of one of a feed's roles.")
    @ApiResponses({
            @ApiResponse(code = 200, message = "The permissions were changed successfully.", response = ActionGroup.class),
            @ApiResponse(code = 404, message = "No feed exists with the specified ID.", response = RestResponseStatus.class) })
    public Response postPermissionsChange(@PathParam("feedId") String feedIdStr, RoleMembershipChange changes) {
        return this.securityService.changeFeedRoleMemberships(feedIdStr, changes).map(m -> Response.ok(m).build())
                .orElseThrow(() -> new WebApplicationException("Either a feed with the ID \"" + feedIdStr
                        + "\" does not exist or it does not have a role the named \"" + changes.getRoleName()
                        + "\"", Status.NOT_FOUND));
    }

    @GET
    @Path("{feedId}/actions/available")
    @Produces(MediaType.APPLICATION_JSON)
    @ApiOperation("Gets the list of available actions that may be permitted or revoked on a feed.")
    @ApiResponses({ @ApiResponse(code = 200, message = "Returns the actions.", response = ActionGroup.class),
            @ApiResponse(code = 404, message = "A feed with the given ID does not exist.", response = RestResponseStatus.class) })
    public Response getAvailableActions(@PathParam("feedId") String feedIdStr) {
        log.debug("Get available actions for feed: {}", feedIdStr);

        return this.securityService.getAvailableFeedActions(feedIdStr).map(g -> Response.ok(g).build()).orElseThrow(
                () -> new WebApplicationException("A feed with the given ID does not exist: " + feedIdStr,
                        Status.NOT_FOUND));
    }

    @GET
    @Path("{feedId}/actions/allowed")
    @Produces(MediaType.APPLICATION_JSON)
    @ApiOperation("Gets the list of actions permitted for the given username and/or groups.")
    @ApiResponses({ @ApiResponse(code = 200, message = "Returns the actions.", response = ActionGroup.class),
            @ApiResponse(code = 404, message = "A feed with the given ID does not exist.", response = RestResponseStatus.class) })
    public Response getAllowedActions(@PathParam("feedId") String feedIdStr,
            @QueryParam("user") Set<String> userNames, @QueryParam("group") Set<String> groupNames) {
        log.debug("Get allowed actions for feed: {}", feedIdStr);

        Set<? extends Principal> users = Arrays.stream(this.securityTransform.asUserPrincipals(userNames))
                .collect(Collectors.toSet());
        Set<? extends Principal> groups = Arrays.stream(this.securityTransform.asGroupPrincipals(groupNames))
                .collect(Collectors.toSet());

        return this.securityService
                .getAllowedFeedActions(feedIdStr,
                        Stream.concat(users.stream(), groups.stream()).collect(Collectors.toSet()))
                .map(g -> Response.ok(g).build()).orElseThrow(() -> new WebApplicationException(
                        "A feed with the given ID does not exist: " + feedIdStr, Status.NOT_FOUND));
    }

    @POST
    @Path("{feedId}/actions/allowed")
    @Consumes(MediaType.APPLICATION_JSON)
    @Produces(MediaType.APPLICATION_JSON)
    @ApiOperation("Updates the permissions for a feed using the supplied permission change request.")
    @ApiResponses({
            @ApiResponse(code = 200, message = "The permissions were changed successfully.", response = ActionGroup.class),
            @ApiResponse(code = 400, message = "The type is not valid.", response = RestResponseStatus.class),
            @ApiResponse(code = 404, message = "No feed exists with the specified ID.", response = RestResponseStatus.class) })
    public Response postPermissionsChange(@PathParam("feedId") String feedIdStr, PermissionsChange changes) {

        return this.securityService.changeFeedPermissions(feedIdStr, changes).map(g -> Response.ok(g).build())
                .orElseThrow(() -> new WebApplicationException(
                        "A feed with the given ID does not exist: " + feedIdStr, Status.NOT_FOUND));
    }

    @GET
    @Path("{feedId}/actions/change")
    @Produces(MediaType.APPLICATION_JSON)
    @ApiOperation("Constructs and returns a permission change request for a set of users/groups containing the actions that the requester may permit or revoke.")
    @ApiResponses({
            @ApiResponse(code = 200, message = "Returns the change request that may be modified by the client and re-posted.", response = PermissionsChange.class),
            @ApiResponse(code = 400, message = "The type is not valid.", response = RestResponseStatus.class),
            @ApiResponse(code = 404, message = "No feed exists with the specified ID.", response = RestResponseStatus.class) })
    public PermissionsChange getAllowedPermissionsChange(@PathParam("feedId") String feedIdStr,
            @QueryParam("type") String changeType, @QueryParam("user") Set<String> userNames,
            @QueryParam("group") Set<String> groupNames) {
        if (StringUtils.isBlank(changeType)) {
            throw new WebApplicationException("The query parameter \"type\" is required", Status.BAD_REQUEST);
        }

        Set<? extends Principal> users = Arrays.stream(this.securityTransform.asUserPrincipals(userNames))
                .collect(Collectors.toSet());
        Set<? extends Principal> groups = Arrays.stream(this.securityTransform.asGroupPrincipals(groupNames))
                .collect(Collectors.toSet());

        return this.securityService
                .createFeedPermissionChange(feedIdStr, ChangeType.valueOf(changeType.toUpperCase()),
                        Stream.concat(users.stream(), groups.stream()).collect(Collectors.toSet()))
                .orElseThrow(() -> new WebApplicationException(
                        "A feed with the given ID does not exist: " + feedIdStr, Status.NOT_FOUND));
    }

    private Response getPage(String processingdttm, int limit, String table) {
        return getPage(processingdttm, limit, table, null);
    }

    private Response getPage(String processingdttm, int limit, String table, String filter) {
        if (limit > MAX_LIMIT) {
            limit = MAX_LIMIT;
        } else if (limit < 1) {
            limit = 1;
        }
        StringBuilder query = new StringBuilder("SELECT * from " + HiveUtils.quoteIdentifier(table)
                + " where processing_dttm = " + HiveUtils.quoteString(processingdttm) + " ");
        if (StringUtils.isNotBlank(filter)) {
            query.append(filter);
        }
        query.append(" limit ").append(limit);
        QueryResult rows = hiveService.query(query.toString());
        return Response.ok(rows.getRows()).build();
    }

    @GET
    @Path("/possible-preconditions")
    @Produces(MediaType.APPLICATION_JSON)
    @ApiOperation("Gets the available preconditions for triggering a feed.")
    @ApiResponses(@ApiResponse(code = 200, message = "Returns the available precondition rules.", response = PreconditionRule.class, responseContainer = "List"))
    public Response getPossiblePreconditions() {
        List<PreconditionRule> conditions = feedManagerPreconditionService.getPossiblePreconditions();
        return Response.ok(conditions).build();
    }

    @GET
    @Path("/{feedId}/sla")
    @Produces(MediaType.APPLICATION_JSON)
    @ApiOperation("Gets the service level agreements referenced by a feed.")
    @ApiResponses({
            @ApiResponse(code = 200, message = "Returns the service level agreements.", response = FeedServiceLevelAgreement.class, responseContainer = "List"),
            @ApiResponse(code = 500, message = "The feed is unavailable.", response = RestResponseStatus.class) })
    public Response getSla(@PathParam("feedId") String feedId) {
        accessController.checkPermission(AccessController.SERVICES,
                FeedServicesAccessControl.ACCESS_SERVICE_LEVEL_AGREEMENTS);
        List<FeedServiceLevelAgreement> sla = serviceLevelAgreementService.getFeedServiceLevelAgreements(feedId);
        if (sla == null) {
            throw new WebApplicationException("No SLAs for the feed were found", Response.Status.NOT_FOUND);
        }
        return Response.ok(sla).build();
    }

    @POST
    @Path("/update-all-datasources")
    @Produces(MediaType.APPLICATION_JSON)
    @ApiOperation("Updates ALL  sources/destinations used for the feed lineage for ALL feeds.  WARNING: This will be an expensive call if you have lots of feeds.  This will remove all existing sources/destinations and revaluate the feed and its template for sources/destinations")
    @ApiResponses({
            @ApiResponse(code = 200, message = "All the feed datasources were updated", response = RestResponseStatus.class), })
    public Response updateAllFeedDataSources() {
        this.accessController.checkPermission(AccessController.SERVICES, MetadataAccessControl.ADMIN_METADATA);
        getMetadataService().updateAllFeedsDatasources();
        return Response.ok(new RestResponseStatus.ResponseStatusBuilder().buildSuccess()).build();
    }

    @POST
    @Path("/{feedId}/update-datasources")
    @Produces(MediaType.APPLICATION_JSON)
    @ApiOperation("Updates a feeds sources/destinations used for the FeedLineage.  This will remove all existing sources/destinations and revaluate the feed and its template for sources/destinations")
    @ApiResponses({
            @ApiResponse(code = 200, message = "the datasources were updated", response = RestResponseStatus.class), })
    public Response updateFeedDatasources(@PathParam("feedId") String feedId) {
        this.accessController.checkPermission(AccessController.SERVICES, MetadataAccessControl.ADMIN_METADATA);
        getMetadataService().updateFeedDatasources(feedId);
        return Response.ok(new RestResponseStatus.ResponseStatusBuilder().buildSuccess()).build();
    }

    @POST
    @Path("/update-feed-lineage-styles")
    @Consumes({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_FORM_URLENCODED })
    @Produces(MediaType.APPLICATION_JSON)
    @ApiOperation("Updates the feed lineage styles.")
    @ApiResponses(@ApiResponse(code = 200, message = "The styles were updated.", response = RestResponseStatus.class))
    public Response updateFeedLineageStyles(Map<String, FeedLineageStyle> styles) {
        accessController.checkPermission(AccessController.SERVICES, FeedServicesAccessControl.ADMIN_FEEDS);

        datasourceService.refreshFeedLineageStyles(styles);
        return Response.ok(new RestResponseStatus.ResponseStatusBuilder().buildSuccess()).build();
    }

    @POST
    @Path("/update-datasource-definitions")
    @Produces(MediaType.APPLICATION_JSON)
    @ApiOperation("Updates the datasource definitions.")
    @ApiResponses(@ApiResponse(code = 200, message = "Returns the updated definitions..", response = DatasourceDefinitions.class))
    public DatasourceDefinitions updateDatasourceDefinitions(DatasourceDefinitions definitions) {
        accessController.checkPermission(AccessController.SERVICES, FeedServicesAccessControl.ADMIN_FEEDS);

        if (definitions != null) {
            Set<DatasourceDefinition> updatedDefinitions = datasourceService
                    .updateDatasourceDefinitions(definitions.getDefinitions());
            if (updatedDefinitions != null) {
                return new DatasourceDefinitions(Lists.newArrayList(updatedDefinitions));
            }
        }
        return new DatasourceDefinitions();
    }

    @POST
    @Path("/{feedId}/upload-file")
    @Consumes(MediaType.MULTIPART_FORM_DATA)
    @Produces(MediaType.APPLICATION_JSON)
    @ApiOperation("Uploads a file to be ingested by a feed.")
    @ApiResponses({ @ApiResponse(code = 200, message = "The file is ready to be ingested."),
            @ApiResponse(code = 500, message = "The file could not be saved.", response = RestResponseStatus.class) })
    public Response uploadFile(@PathParam("feedId") String feedId,
            @FormDataParam("file") InputStream fileInputStream,
            @FormDataParam("file") FormDataContentDisposition fileMetaData) throws Exception {

        FeedMetadata feed = getMetadataService().getFeedById(feedId, false);
        // Derive path and file
        List<NifiProperty> properties = feed.getProperties();
        String dropzone = null;
        String regexFileFilter = null;

        for (NifiProperty property : properties) {

            if (property.getProcessorType().equals("org.apache.nifi.processors.standard.GetFile")) {
                if (property.getKey().equals("File Filter")) {
                    regexFileFilter = property.getValue();
                } else if (property.getKey().equals("Input Directory")) {
                    dropzone = property.getValue();
                }
            }
        }
        if (StringUtils.isEmpty(regexFileFilter) || StringUtils.isEmpty(dropzone)) {
            throw new IOException("Unable to upload file with empty dropzone and file");
        }
        File tempTarget = File.createTempFile("kylo-upload", "");
        String fileName = "";
        try {
            Generex fileNameGenerator = new Generex(regexFileFilter);
            fileName = fileNameGenerator.random();

            // Cleanup oddball characters generated by generex
            fileName = fileName.replaceAll("[^A-Za-z0-9\\.\\_\\+\\%\\-\\|]+", "\\.");
            java.nio.file.Path dropZoneTarget = Paths.get(dropzone, fileName);
            File dropZoneFile = dropZoneTarget.toFile();
            if (dropZoneFile.exists()) {
                throw new IOException("File with the name [" + fileName + "] already exists in [" + dropzone + "]");
            }

            Files.copy(fileInputStream, tempTarget.toPath(), StandardCopyOption.REPLACE_EXISTING);
            Files.move(tempTarget.toPath(), dropZoneTarget);

            // Set read, write
            dropZoneFile.setReadable(true);
            dropZoneFile.setWritable(true);

        } catch (AccessDeniedException e) {
            String errTemplate = "Permission denied attempting to write file [%s] to [%s]. Check with system administrator to ensure this application has write permissions to folder";
            String err = String.format(errTemplate, fileName, dropzone);
            log.error(err);
            throw new InternalServerErrorException(err);

        } catch (Exception e) {
            String errTemplate = "Unexpected exception writing file [%s] to [%s].";
            String err = String.format(errTemplate, fileName, dropzone);
            log.error(err);
            throw new InternalServerErrorException(err);
        }
        return Response.ok("").build();
    }

    private PageRequest pageRequest(Integer start, Integer limit, String sort) {
        if (StringUtils.isNotBlank(sort)) {
            Sort.Direction dir = Sort.Direction.ASC;
            if (sort.startsWith("-")) {
                dir = Sort.Direction.DESC;
                sort = sort.substring(1);
            }
            return new PageRequest((start / limit), limit, dir, sort);
        } else {
            return new PageRequest((start / limit), limit);
        }
    }
}