Java tutorial
/* * Copyright 2016 ANI Technologies Pvt. Ltd. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.olacabs.fabric.processors.httpwriter; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Strings; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.olacabs.fabric.compute.ProcessingContext; import com.olacabs.fabric.compute.processor.InitializationException; import com.olacabs.fabric.compute.processor.ProcessingException; import com.olacabs.fabric.compute.processor.StreamingProcessor; import com.olacabs.fabric.compute.util.ComponentPropertyReader; import com.olacabs.fabric.model.common.ComponentMetadata; import com.olacabs.fabric.model.event.Event; import com.olacabs.fabric.model.event.EventSet; import com.olacabs.fabric.model.processor.Processor; import com.olacabs.fabric.model.processor.ProcessorType; import lombok.Data; import lombok.EqualsAndHashCode; import lombok.extern.slf4j.Slf4j; import org.apache.http.HttpEntity; import org.apache.http.HttpEntityEnclosingRequest; import org.apache.http.HttpRequestInterceptor; import org.apache.http.HttpResponse; import org.apache.http.auth.AuthScope; import org.apache.http.auth.AuthState; import org.apache.http.auth.Credentials; import org.apache.http.auth.UsernamePasswordCredentials; import org.apache.http.client.CredentialsProvider; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpGet; import org.apache.http.client.methods.HttpPost; import org.apache.http.client.methods.HttpPut; import org.apache.http.client.methods.HttpRequestBase; import org.apache.http.client.protocol.HttpClientContext; import org.apache.http.entity.ByteArrayEntity; import org.apache.http.impl.auth.BasicScheme; import org.apache.http.impl.client.BasicCredentialsProvider; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClientBuilder; import org.apache.http.impl.client.HttpClients; import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; import org.apache.http.util.EntityUtils; import java.io.IOException; import java.util.HashMap; import java.util.Map; import java.util.Properties; /** * A processor that sends events to an http end point. */ @EqualsAndHashCode(callSuper = true) @VisibleForTesting @Slf4j @Data @Processor(namespace = "global", name = "http-writer", version = "1.0.0", description = "A processor that sends events to an HTTP end point!!", cpu = 0.5, memory = 512, processorType = ProcessorType.EVENT_DRIVEN, requiredProperties = { "endpoint_url", "bulk_supported", "http_method", "auth_enabled" }, optionalProperties = { "headers", "should_publish_response", "pool_size", "auth_configuration" }) public class HttpWriter extends StreamingProcessor { private static final int DEFAULT_POOL_SIZE = 10; private int poolSize; private ObjectMapper mapper; private String headerString; private PoolingHttpClientConnectionManager manager; private String endPointUrl; private Map<String, String> headers; private boolean bulkSupported; private String httpMethod; private CloseableHttpClient client; private HttpClientBuilder httpClientBuilder; private boolean shouldPublishResponse; private Boolean authEnabled; /** * Starts the HttpClient and Manager. */ protected void start(AuthConfiguration authConfiguration, Boolean authEnabledParam) throws InitializationException { this.manager = new PoolingHttpClientConnectionManager(); manager.setMaxTotal(poolSize); manager.setDefaultMaxPerRoute(poolSize); //client = HttpClients.custom().setConnectionManager(manager).build(); httpClientBuilder = HttpClients.custom(); httpClientBuilder.setConnectionManager(manager); if (authEnabledParam) { setAuth(authConfiguration, httpClientBuilder); } client = httpClientBuilder.build(); } /** * Stop the HttpClient and Manager. * * @throws IOException */ private void stop() throws IOException { client.close(); manager.close(); } @Override protected EventSet consume(ProcessingContext processingContext, EventSet eventSet) throws ProcessingException { log.debug("Method - {}", this.getHttpMethod().toLowerCase()); return this.handleRequest(eventSet, this.getHttpMethod().toLowerCase()); } private EventSet handleRequest(EventSet eventSet, String httpMethodType) throws ProcessingException { ImmutableList.Builder<Object> builder = ImmutableList.builder(); EventSet.EventFromEventBuilder eventSetBuilder = EventSet.eventFromEventBuilder(); HttpRequestBase request = null; boolean entityEnclosable = false; switch (httpMethodType) { case "get": request = createHttpGet(); break; case "post": request = createHttpPost(); entityEnclosable = true; break; case "put": request = createHttpPut(); entityEnclosable = true; break; default: log.error("Method not recognized...{}", this.getHttpMethod()); } request.setHeader("Content-Type", "application/json"); if (null != this.getHeaders()) { this.getHeaders().forEach(request::setHeader); } CloseableHttpResponse response = null; log.debug("Handling Put Request"); for (Event event : eventSet.getEvents()) { try { log.debug("Event in json format : {}", event.getJsonNode()); if (!isBulkSupported()) { if (entityEnclosable) { ((HttpEntityEnclosingRequest) request) .setEntity(new ByteArrayEntity((byte[]) event.getData())); } response = getClient().execute(request); handleResponse(response); if (shouldPublishResponse) { Event e = Event.builder() .jsonNode(getMapper().convertValue( createResponseMap(event, response, httpMethodType), JsonNode.class)) .build(); e.setProperties(event.getProperties()); eventSetBuilder.isAggregate(false).partitionId(eventSet.getPartitionId()).event(e); } } else { builder.add(event.getData()); } } catch (Exception e) { log.error("Error processing data", e); throw new ProcessingException(); } finally { close(response); } } if (isBulkSupported()) { try { log.debug("Making Bulk call as bulk is supported"); if (entityEnclosable) { ((HttpEntityEnclosingRequest) request) .setEntity(new ByteArrayEntity(getMapper().writeValueAsBytes(builder.build()))); } response = client.execute(request); handleResponse(response); } catch (Exception e) { log.error("Exception", e); throw new ProcessingException(); } finally { close(response); } } else { if (shouldPublishResponse) { return eventSetBuilder.build(); } } return null; } private void handleResponse(HttpResponse response) throws IOException, ProcessingException { if (null != response) { log.debug("Received response {}", response.getStatusLine().getStatusCode()); } else { log.error("No response received from downstream system"); } } private HttpPut createHttpPut() { HttpPut put = new HttpPut(this.getEndPointUrl()); return put; } private HttpGet createHttpGet() { HttpGet get = new HttpGet(this.getEndPointUrl()); return get; } private void close(CloseableHttpResponse response) { if (null != response) { try { response.close(); } catch (IOException e) { log.error("Error closing http client: ", e); } } } private HttpPost createHttpPost() { log.debug("Creating http post request"); HttpPost post = new HttpPost(this.getEndPointUrl()); return post; } @Override public void initialize(String instanceId, Properties globalProperties, Properties properties, ComponentMetadata componentMetadata) throws InitializationException { this.endPointUrl = ComponentPropertyReader.readString(properties, globalProperties, "endpoint_url", instanceId, componentMetadata); this.headerString = ComponentPropertyReader.readString(properties, globalProperties, "headers", instanceId, componentMetadata); this.httpMethod = ComponentPropertyReader.readString(properties, globalProperties, "http_method", instanceId, componentMetadata); this.shouldPublishResponse = ComponentPropertyReader.readBoolean(properties, globalProperties, "should_publish_response", instanceId, componentMetadata, false); this.mapper = new ObjectMapper(); if (!Strings.isNullOrEmpty(headerString)) { TypeReference<HashMap<String, String>> typeReference = new TypeReference<HashMap<String, String>>() { }; try { this.headers = getMapper().readValue(headerString, typeReference); } catch (Exception e) { log.error("Error while converting headers", e); throw new InitializationException(); } } this.bulkSupported = ComponentPropertyReader.readBoolean(properties, globalProperties, "bulk_supported", instanceId, componentMetadata, false); this.poolSize = ComponentPropertyReader.readInteger(properties, globalProperties, "pool_size", instanceId, componentMetadata, DEFAULT_POOL_SIZE); this.authEnabled = ComponentPropertyReader.readBoolean(properties, globalProperties, "auth_enabled", instanceId, componentMetadata, false); try { AuthConfiguration authConfiguration = null; if (authEnabled) { authConfiguration = getMapper() .readValue( ComponentPropertyReader.readString(properties, globalProperties, "auth_configuration", instanceId, componentMetadata), AuthConfiguration.class); } start(authConfiguration, authEnabled); } catch (Exception e) { log.error("Unable to start the httpclient - {}", e); throw new InitializationException(); } } /** * creates a response map that can be set in the new event set. * * @param event event to be added in the response * @param response http response * @param httpMethodType the http method type * @return req */ private Map<String, Object> createResponseMap(Event event, CloseableHttpResponse response, String httpMethodType) throws IOException { return ImmutableMap.of("SourceEvent", event.getJsonNode(), "HttpMethod", httpMethodType, "HttpResponseCode", response.getStatusLine().getStatusCode(), "HttpResponse", getResponseAsJson(response)); } private JsonNode getResponseAsJson(CloseableHttpResponse response) throws IOException { try { if (null != response) { final HttpEntity entity = response.getEntity(); if (null != entity) { final String responseStr = EntityUtils.toString(entity); try { return (!Strings.isNullOrEmpty(responseStr)) ? getMapper().readTree(responseStr) : null; } catch (final Exception e) { return getMapper().createObjectNode().put("Response", responseStr); } } } } catch (Exception e) { log.error("Exception while parsing the response - {}", e); } return getMapper().createObjectNode(); } @Override public void destroy() { try { stop(); } catch (Exception e) { log.error("Error while closing the connection - {}", e); } } private void setAuth(AuthConfiguration authConfiguration, HttpClientBuilder builder) throws InitializationException { if (!Strings.isNullOrEmpty(authConfiguration.getUsername())) { Credentials credentials = new UsernamePasswordCredentials(authConfiguration.getUsername(), authConfiguration.getPassword()); CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); credentialsProvider.setCredentials(new AuthScope(AuthScope.ANY_HOST, AuthScope.ANY_PORT), credentials); builder.addInterceptorFirst((HttpRequestInterceptor) (request, context) -> { AuthState authState = (AuthState) context.getAttribute(HttpClientContext.TARGET_AUTH_STATE); if (authState.getAuthScheme() == null) { //log.debug("SETTING CREDS"); //log.info("Preemptive AuthState: {}", authState); authState.update(new BasicScheme(), credentials); } }); } else { log.error("Username can't be blank for basic auth."); throw new InitializationException("Username blank for basic auth"); } } }