List of usage examples for org.apache.http.client.utils URIBuilder getScheme
public String getScheme()
From source file:org.apache.ambari.server.controller.ganglia.GangliaPropertyProviderTest.java
@Test public void testPopulateResources() throws Exception { TestStreamProvider streamProvider = new TestStreamProvider("temporal_ganglia_data.txt"); TestGangliaHostProvider hostProvider = new TestGangliaHostProvider(); Map<String, Map<String, PropertyInfo>> gangliaPropertyIds = PropertyHelper .getGangliaPropertyIds(Resource.Type.HostComponent); GangliaPropertyProvider propertyProvider = new GangliaHostComponentPropertyProvider(gangliaPropertyIds, streamProvider, configuration, hostProvider, CLUSTER_NAME_PROPERTY_ID, HOST_NAME_PROPERTY_ID, COMPONENT_NAME_PROPERTY_ID); // namenode/*from w w w. j av a 2s .c o m*/ Resource resource = new ResourceImpl(Resource.Type.HostComponent); resource.setProperty(HOST_NAME_PROPERTY_ID, "domU-12-31-39-0E-34-E1.compute-1.internal"); resource.setProperty(COMPONENT_NAME_PROPERTY_ID, "DATANODE"); // only ask for one property Map<String, TemporalInfo> temporalInfoMap = new HashMap<String, TemporalInfo>(); temporalInfoMap.put(PROPERTY_ID, new TemporalInfoImpl(10L, 20L, 1L)); Request request = PropertyHelper.getReadRequest(Collections.singleton(PROPERTY_ID), temporalInfoMap); Assert.assertEquals(1, propertyProvider.populateResources(Collections.singleton(resource), request, null).size()); String expected = (configuration.isGangliaSSL() ? "https" : "http") + "://domU-12-31-39-0E-34-E1.compute-1.internal/cgi-bin/rrd.py?c=HDPDataNode%2CHDPSlaves&h=domU-12-31-39-0E-34-E1.compute-1.internal&m=jvm.metrics.gcCount&s=10&e=20&r=1"; Assert.assertEquals(expected, streamProvider.getLastSpec()); Assert.assertEquals(3, PropertyHelper.getProperties(resource).size()); Assert.assertNotNull(resource.getPropertyValue(PROPERTY_ID)); // tasktracker resource = new ResourceImpl(Resource.Type.HostComponent); resource.setProperty(HOST_NAME_PROPERTY_ID, "domU-12-31-39-0E-34-E1.compute-1.internal"); resource.setProperty(COMPONENT_NAME_PROPERTY_ID, "TASKTRACKER"); // only ask for one property temporalInfoMap = new HashMap<String, TemporalInfo>(); Set<String> properties = new HashSet<String>(); String shuffle_exceptions_caught = PropertyHelper.getPropertyId("metrics/mapred/shuffleOutput", "shuffle_exceptions_caught"); String shuffle_failed_outputs = PropertyHelper.getPropertyId("metrics/mapred/shuffleOutput", "shuffle_failed_outputs"); String shuffle_output_bytes = PropertyHelper.getPropertyId("metrics/mapred/shuffleOutput", "shuffle_output_bytes"); String shuffle_success_outputs = PropertyHelper.getPropertyId("metrics/mapred/shuffleOutput", "shuffle_success_outputs"); properties.add(shuffle_exceptions_caught); properties.add(shuffle_failed_outputs); properties.add(shuffle_output_bytes); properties.add(shuffle_success_outputs); request = PropertyHelper.getReadRequest(properties, temporalInfoMap); temporalInfoMap.put(shuffle_exceptions_caught, new TemporalInfoImpl(10L, 20L, 1L)); temporalInfoMap.put(shuffle_failed_outputs, new TemporalInfoImpl(10L, 20L, 1L)); temporalInfoMap.put(shuffle_output_bytes, new TemporalInfoImpl(10L, 20L, 1L)); temporalInfoMap.put(shuffle_success_outputs, new TemporalInfoImpl(10L, 20L, 1L)); Assert.assertEquals(1, propertyProvider.populateResources(Collections.singleton(resource), request, null).size()); List<String> metricsRegexes = new ArrayList<String>(); metricsRegexes.add("metrics/mapred/shuffleOutput/shuffle_exceptions_caught"); metricsRegexes.add("metrics/mapred/shuffleOutput/shuffle_failed_outputs"); metricsRegexes.add("metrics/mapred/shuffleOutput/shuffle_output_bytes"); metricsRegexes.add("metrics/mapred/shuffleOutput/shuffle_success_outputs"); String metricsList = getMetricsRegexes(metricsRegexes, gangliaPropertyIds, "TASKTRACKER"); URIBuilder expectedUri = new URIBuilder(); expectedUri.setScheme((configuration.isGangliaSSL() ? "https" : "http")); expectedUri.setHost("domU-12-31-39-0E-34-E1.compute-1.internal"); expectedUri.setPath("/cgi-bin/rrd.py"); expectedUri.setParameter("c", "HDPTaskTracker,HDPSlaves"); expectedUri.setParameter("h", "domU-12-31-39-0E-34-E1.compute-1.internal"); expectedUri.setParameter("m", metricsList); expectedUri.setParameter("s", "10"); expectedUri.setParameter("e", "20"); expectedUri.setParameter("r", "1"); URIBuilder actualUri = new URIBuilder(streamProvider.getLastSpec()); Assert.assertEquals(expectedUri.getScheme(), actualUri.getScheme()); Assert.assertEquals(expectedUri.getHost(), actualUri.getHost()); Assert.assertEquals(expectedUri.getPath(), actualUri.getPath()); Assert.assertTrue(isUrlParamsEquals(actualUri, expectedUri)); Assert.assertEquals(6, PropertyHelper.getProperties(resource).size()); Assert.assertNotNull(resource.getPropertyValue(shuffle_exceptions_caught)); Number[][] dataPoints = (Number[][]) resource.getPropertyValue(shuffle_exceptions_caught); Assert.assertEquals(106, dataPoints.length); for (int i = 0; i < dataPoints.length; ++i) { Assert.assertEquals(i >= 10 && i < 20 ? 7 : 0.0, dataPoints[i][0]); Assert.assertEquals(360 * i + 1358434800, dataPoints[i][1]); } Assert.assertNotNull(resource.getPropertyValue(shuffle_failed_outputs)); Assert.assertNotNull(resource.getPropertyValue(shuffle_output_bytes)); Assert.assertNotNull(resource.getPropertyValue(shuffle_success_outputs)); }
From source file:org.apache.ambari.server.controller.ganglia.GangliaPropertyProviderTest.java
@Test public void testPopulateResources__LargeNumberOfHostResources() throws Exception { TestStreamProvider streamProvider = new TestStreamProvider("temporal_ganglia_data.txt"); TestGangliaHostProvider hostProvider = new TestGangliaHostProvider(); GangliaPropertyProvider propertyProvider = new GangliaHostPropertyProvider( PropertyHelper.getGangliaPropertyIds(Resource.Type.Host), streamProvider, configuration, hostProvider, CLUSTER_NAME_PROPERTY_ID, HOST_NAME_PROPERTY_ID); Set<Resource> resources = new HashSet<Resource>(); StringBuilder hostsList = new StringBuilder(); for (int i = 0; i < 150; ++i) { Resource resource = new ResourceImpl(Resource.Type.Host); resource.setProperty(HOST_NAME_PROPERTY_ID, "host" + i); resources.add(resource);/*from ww w . java 2 s . c o m*/ if (hostsList.length() != 0) hostsList.append("," + "host" + i); else hostsList.append("host" + i); } // only ask for one property Map<String, TemporalInfo> temporalInfoMap = new HashMap<String, TemporalInfo>(); temporalInfoMap.put(PROPERTY_ID, new TemporalInfoImpl(10L, 20L, 1L)); Request request = PropertyHelper.getReadRequest(Collections.singleton(PROPERTY_ID), temporalInfoMap); Assert.assertEquals(150, propertyProvider.populateResources(resources, request, null).size()); URIBuilder expectedUri = new URIBuilder(); expectedUri.setScheme((configuration.isGangliaSSL() ? "https" : "http")); expectedUri.setHost("domU-12-31-39-0E-34-E1.compute-1.internal"); expectedUri.setPath("/cgi-bin/rrd.py"); expectedUri.setParameter("c", "HDPJobTracker,HDPHBaseMaster,HDPKafka,HDPResourceManager,HDPFlumeServer,HDPSlaves,HDPHistoryServer,HDPJournalNode,HDPTaskTracker,HDPHBaseRegionServer,HDPNameNode"); expectedUri.setParameter("h", hostsList.toString()); expectedUri.setParameter("m", "jvm.metrics.gcCount"); expectedUri.setParameter("s", "10"); expectedUri.setParameter("e", "20"); expectedUri.setParameter("r", "1"); URIBuilder actualUri = new URIBuilder(streamProvider.getLastSpec()); Assert.assertEquals(expectedUri.getScheme(), actualUri.getScheme()); Assert.assertEquals(expectedUri.getHost(), actualUri.getHost()); Assert.assertEquals(expectedUri.getPath(), actualUri.getPath()); Assert.assertTrue(isUrlParamsEquals(actualUri, expectedUri)); }
From source file:nl.nn.adapterframework.http.HttpSenderBase.java
@Override public String sendMessageWithTimeoutGuarded(String correlationID, String message, ParameterResolutionContext prc) throws SenderException, TimeOutException { ParameterValueList pvl = null;//from ww w . ja va 2s.c o m try { if (prc != null && paramList != null) { pvl = prc.getValues(paramList); } } catch (ParameterException e) { throw new SenderException( getLogPrefix() + "Sender [" + getName() + "] caught exception evaluating parameters", e); } HttpHost httpTarget; URIBuilder uri; HttpRequestBase httpRequestBase; try { if (urlParameter != null) { String url = (String) pvl.getParameterValue(getUrlParam()).getValue(); uri = getURI(url); } else { uri = staticUri; } httpTarget = new HttpHost(uri.getHost(), getPort(uri), uri.getScheme()); Map<String, String> headersParamsMap = new HashMap<String, String>(); if (headersParams != null) { StringTokenizer st = new StringTokenizer(headersParams, ","); while (st.hasMoreElements()) { headersParamsMap.put(st.nextToken(), null); } } if (isEncodeMessages()) { message = URLEncoder.encode(message, getCharSet()); } httpRequestBase = getMethod(uri, message, pvl, headersParamsMap, (prc == null) ? null : prc.getSession()); if (httpRequestBase == null) throw new MethodNotSupportedException( "could not find implementation for method [" + getMethodType() + "]"); if (StringUtils.isNotEmpty(getContentType())) { httpRequestBase.setHeader("Content-Type", getContentType()); } if (credentials != null && !StringUtils.isEmpty(credentials.getUsername())) { AuthCache authCache = httpClientContext.getAuthCache(); if (authCache == null) authCache = new BasicAuthCache(); if (authCache.get(httpTarget) == null) authCache.put(httpTarget, new BasicScheme()); httpClientContext.setAuthCache(authCache); } log.info(getLogPrefix() + "configured httpclient for host [" + uri.getHost() + "]"); } catch (Exception e) { throw new SenderException(e); } String result = null; int statusCode = -1; int count = getMaxExecuteRetries(); String msg = null; while (count-- >= 0 && statusCode == -1) { try { log.debug(getLogPrefix() + "executing method [" + httpRequestBase.getRequestLine() + "]"); HttpResponse httpResponse = getHttpClient().execute(httpTarget, httpRequestBase, httpClientContext); log.debug(getLogPrefix() + "executed method"); HttpResponseHandler responseHandler = new HttpResponseHandler(httpResponse); StatusLine statusline = httpResponse.getStatusLine(); statusCode = statusline.getStatusCode(); if (StringUtils.isNotEmpty(getResultStatusCodeSessionKey()) && prc != null) { prc.getSession().put(getResultStatusCodeSessionKey(), Integer.toString(statusCode)); } // Only give warnings for 4xx (client errors) and 5xx (server errors) if (statusCode >= 400 && statusCode < 600) { log.warn(getLogPrefix() + "status [" + statusline.toString() + "]"); } else { log.debug(getLogPrefix() + "status [" + statusCode + "]"); } result = extractResult(responseHandler, prc); log.debug(getLogPrefix() + "retrieved result [" + result + "]"); } catch (ClientProtocolException e) { StringBuilder msgBuilder = new StringBuilder(getLogPrefix() + "httpException with"); if (e.getMessage() != null) { msg = e.getMessage(); msgBuilder.append(" message [" + msg + "]"); } Throwable throwable = e.getCause(); if (throwable != null) { msgBuilder.append(" cause [" + throwable.toString() + "]"); } msgBuilder.append(" executeRetries left [" + count + "]"); log.warn(msgBuilder.toString()); } catch (IOException e) { httpRequestBase.abort(); if (e instanceof SocketTimeoutException) { throw new TimeOutException(e); } throw new SenderException(e); } finally { // By forcing the use of the HttpResponseHandler the resultStream // will automatically be closed when it has been read. // See HttpResponseHandler and ReleaseConnectionAfterReadInputStream. // We cannot close the connection as the response might be kept // in a sessionKey for later use in the pipeline. // // IMPORTANT: It is possible that poorly written implementations // wont read or close the response. // This will cause the connection to become stale.. } } if (statusCode == -1) { if (msg != null && StringUtils.contains(msg.toUpperCase(), "TIMEOUTEXCEPTION")) { //java.net.SocketTimeoutException: Read timed out throw new TimeOutException("Failed to recover from timeout exception"); } throw new SenderException("Failed to recover from exception"); } if (isXhtml() && StringUtils.isNotEmpty(result)) { result = XmlUtils.skipDocTypeDeclaration(result.trim()); if (result.startsWith("<html>") || result.startsWith("<html ")) { CleanerProperties props = new CleanerProperties(); HtmlCleaner cleaner = new HtmlCleaner(props); TagNode tagNode = cleaner.clean(result); result = new SimpleXmlSerializer(props).getAsString(tagNode); if (transformerPool != null) { log.debug(getLogPrefix() + " transforming result [" + result + "]"); ParameterResolutionContext prc_xslt = new ParameterResolutionContext(result, null, true); try { result = transformerPool.transform(prc_xslt.getInputSource(true), null); } catch (Exception e) { throw new SenderException("Exception on transforming input", e); } } } } return result; }
From source file:com.sonicle.webtop.calendar.CalendarManager.java
public ProbeCalendarRemoteUrlResult probeCalendarRemoteUrl(Calendar.Provider provider, URI url, String username, String password) throws WTException { if (!Calendar.isProviderRemote(provider)) { throw new WTException("Provider is not remote (webcal or CalDAV) [{}]", EnumUtils.toSerializedName(provider)); }//from w w w.j a va2 s . co m try { if (Calendar.Provider.WEBCAL.equals(provider)) { URIBuilder builder = new URIBuilder(url); if (StringUtils.equalsIgnoreCase(builder.getScheme(), "webcal")) { builder.setScheme("http"); // Force http scheme } if (!StringUtils.isBlank(username) && !StringUtils.isBlank(username)) { builder.setUserInfo(username, password); } URI newUrl = URIUtils.buildQuietly(builder); HttpClient httpCli = null; try { logger.debug("Checking remote calendar URL [{}]", newUrl.toString()); httpCli = HttpClientUtils.createBasicHttpClient(HttpClientUtils.configureSSLAcceptAll(), newUrl); return HttpClientUtils.exists(httpCli, newUrl) ? new ProbeCalendarRemoteUrlResult(FilenameUtils.getBaseName(newUrl.getPath())) : null; } finally { HttpClientUtils.closeQuietly(httpCli); } } else if (Calendar.Provider.CALDAV.equals(provider)) { CalDav dav = getCalDav(username, password); try { DavCalendar dcal = dav.getCalendar(url.toString()); return (dcal != null) ? new ProbeCalendarRemoteUrlResult(dcal.getDisplayName()) : null; } catch (DavException ex) { logger.error("DAV error", ex); return null; } } else { throw new WTException("Unsupported provider"); } } catch (IOException ex) { throw new WTException(ex, "Unable to check URL [{0}]", url.toString()); } }
From source file:com.sonicle.webtop.calendar.CalendarManager.java
public void syncRemoteCalendar(int calendarId, boolean full) throws WTException { final UserProfile.Data udata = WT.getUserData(getTargetProfileId()); final ICalendarInput icalInput = new ICalendarInput(udata.getTimeZone()); final String PENDING_KEY = String.valueOf(calendarId); CalendarDAO calDao = CalendarDAO.getInstance(); Connection con = null;/*from www . j a v a 2 s . c om*/ if (pendingRemoteCalendarSyncs.putIfAbsent(PENDING_KEY, RunContext.getRunProfileId()) != null) { throw new ConcurrentSyncException("Sync activity is already running [{}, {}]", calendarId, RunContext.getRunProfileId()); } try { //checkRightsOnCalendarFolder(calendarId, "READ"); con = WT.getConnection(SERVICE_ID, false); Calendar cal = ManagerUtils.createCalendar(calDao.selectById(con, calendarId)); if (cal == null) throw new WTException("Calendar not found [{0}]", calendarId); if (!Calendar.Provider.WEBCAL.equals(cal.getProvider()) && !Calendar.Provider.CALDAV.equals(cal.getProvider())) { throw new WTException("Specified calendar is not remote (webcal or CalDAV) [{0}]", calendarId); } // Force a full update if last-sync date is null if (cal.getRemoteSyncTimestamp() == null) full = true; CalendarRemoteParameters params = LangUtils.deserialize(cal.getParameters(), CalendarRemoteParameters.class); if (params == null) throw new WTException("Unable to deserialize remote parameters"); if (params.url == null) throw new WTException("Remote URL is undefined"); if (Calendar.Provider.WEBCAL.equals(cal.getProvider())) { final String PREFIX = "webcal-"; File tempFile = null; URIBuilder builder = new URIBuilder(params.url); if (StringUtils.equalsIgnoreCase(builder.getScheme(), "webcal")) { builder.setScheme("http"); // Force http scheme } if (!StringUtils.isBlank(params.username) && !StringUtils.isBlank(params.username)) { builder.setUserInfo(params.username, params.password); } URI newUrl = URIUtils.buildQuietly(builder); try { final DateTime newLastSync = DateTimeUtils.now(); tempFile = WT.createTempFile(PREFIX, null); // Retrieve webcal content (iCalendar) from the specified URL // and save it locally logger.debug("Downloading iCalendar file from URL [{}]", newUrl); HttpClient httpCli = null; FileOutputStream os = null; try { httpCli = HttpClientUtils.createBasicHttpClient(HttpClientUtils.configureSSLAcceptAll(), newUrl); os = new FileOutputStream(tempFile); HttpClientUtils.writeContent(httpCli, newUrl, os); } catch (IOException ex) { throw new WTException(ex, "Unable to retrieve webcal [{0}]", newUrl); } finally { IOUtils.closeQuietly(os); HttpClientUtils.closeQuietly(httpCli); } logger.debug("Saved to temp file [{}]", tempFile.getName()); // Parse downloaded iCalendar logger.debug("Parsing downloaded iCalendar file"); net.fortuna.ical4j.model.Calendar ical = null; FileInputStream is = null; try { is = new FileInputStream(tempFile); ICalendarUtils.relaxParsingAndCompatibility(); ical = ICalendarUtils.parse(is); //TODO: add support to FILENAME property (Google https://github.com/ical4j/ical4j/issues/69) } catch (IOException | ParserException ex) { throw new WTException(ex, "Unable to read webcal"); } finally { IOUtils.closeQuietly(os); } icalInput.withIncludeVEventSourceInOutput(true); ArrayList<EventInput> input = icalInput.fromICalendarFile(ical, null); logger.debug("Found {} events", input.size()); Map<String, VEventHrefSync> syncByHref = null; if (full) { logger.debug("Cleaning up calendar [{}]", calendarId); doEventsDeleteByCalendar(con, calendarId, false); } else { EventDAO evtDao = EventDAO.getInstance(); syncByHref = evtDao.viewHrefSyncDataByCalendar(con, calendarId); } // Inserts/Updates data... logger.debug("Inserting/Updating events..."); try { String autoUidPrefix = DigestUtils.md5Hex(newUrl.toString()); // auto-gen base prefix in case of missing UID HashSet<String> hrefs = new HashSet<>(); HashMap<String, OEvent> cache = new HashMap<>(); int i = 0; for (EventInput ei : input) { if (StringUtils.isBlank(ei.event.getPublicUid())) { String autoUid = autoUidPrefix + "-" + i; ei.event.setPublicUid(autoUid); logger.trace("Missing UID: using auto-gen value. [{}]", autoUid); } String href = ManagerUtils.buildHref(ei.event.getPublicUid()); //if (logger.isTraceEnabled()) logger.trace("{}", ICalendarUtils.print(ICalendarUtils.getVEvent(devt.getCalendar()))); if (hrefs.contains(href)) { logger.trace("Event duplicated. Skipped! [{}]", href); continue; } boolean skip = false; Integer matchingEventId = null; String eiHash = DigestUtils.md5Hex(ei.sourceEvent.toString()); if (syncByHref != null) { // Only if... (!full) see above! VEventHrefSync hrefSync = syncByHref.remove(href); if (hrefSync != null) { // Href found -> maybe updated item if (!StringUtils.equals(hrefSync.getEtag(), eiHash)) { matchingEventId = hrefSync.getEventId(); logger.trace("Event updated [{}, {}]", href, eiHash); } else { skip = true; logger.trace("Event not modified [{}, {}]", href, eiHash); } } else { // Href not found -> added item logger.trace("Event newly added [{}, {}]", href, eiHash); } } if (!skip) { ei.event.setCalendarId(calendarId); ei.event.setHref(href); ei.event.setEtag(eiHash); if (matchingEventId != null) { ei.event.setEventId(matchingEventId); boolean updated = doEventInputUpdate(con, cache, ei); if (!updated) throw new WTException("Event not found [{}]", ei.event.getEventId()); } else { doEventInputInsert(con, cache, ei); } } hrefs.add(href); // Marks as processed! } if (syncByHref != null) { // Only if... (!full) see above! // Remaining hrefs -> deleted items for (VEventHrefSync hrefSync : syncByHref.values()) { logger.trace("Event deleted [{}]", hrefSync.getHref()); doEventDelete(con, hrefSync.getEventId(), false); } } cache.clear(); calDao.updateRemoteSyncById(con, calendarId, newLastSync, null); DbUtils.commitQuietly(con); } catch (Exception ex) { DbUtils.rollbackQuietly(con); throw new WTException(ex, "Error importing iCalendar"); } } finally { if (tempFile != null) { logger.debug("Removing temp file [{}]", tempFile.getName()); WT.deleteTempFile(tempFile); } } } else if (Calendar.Provider.CALDAV.equals(cal.getProvider())) { CalDav dav = getCalDav(params.username, params.password); try { DavCalendar dcal = dav.getCalendarSyncToken(params.url.toString()); if (dcal == null) throw new WTException("DAV calendar not found"); final boolean syncIsSupported = !StringUtils.isBlank(dcal.getSyncToken()); final DateTime newLastSync = DateTimeUtils.now(); if (!full && (syncIsSupported && !StringUtils.isBlank(cal.getRemoteSyncTag()))) { // Partial update using SYNC mode String newSyncToken = dcal.getSyncToken(); logger.debug("Querying CalDAV endpoint for changes [{}, {}]", params.url.toString(), cal.getRemoteSyncTag()); List<DavSyncStatus> changes = dav.getCalendarChanges(params.url.toString(), cal.getRemoteSyncTag()); logger.debug("Returned {} items", changes.size()); try { if (!changes.isEmpty()) { EventDAO evtDao = EventDAO.getInstance(); Map<String, List<Integer>> eventIdsByHref = evtDao.selectHrefsByByCalendar(con, calendarId); // Process changes... logger.debug("Processing changes..."); HashSet<String> hrefs = new HashSet<>(); for (DavSyncStatus change : changes) { String href = FilenameUtils.getName(change.getPath()); //String href = change.getPath(); if (DavUtil.HTTP_SC_TEXT_OK.equals(change.getResponseStatus())) { hrefs.add(href); } else { // Event deleted List<Integer> eventIds = eventIdsByHref.get(href); Integer eventId = (eventIds != null) ? eventIds.get(eventIds.size() - 1) : null; if (eventId == null) { logger.warn("Deletion not possible. Event path not found [{}]", PathUtils.concatPaths(dcal.getPath(), FilenameUtils.getName(href))); continue; } doEventDelete(con, eventId, false); } } // Retrieves events list from DAV endpoint (using multiget) logger.debug("Retrieving inserted/updated events [{}]", hrefs.size()); Collection<String> paths = hrefs.stream().map( href -> PathUtils.concatPaths(dcal.getPath(), FilenameUtils.getName(href))) .collect(Collectors.toList()); List<DavCalendarEvent> devts = dav.listCalendarEvents(params.url.toString(), paths); //List<DavCalendarEvent> devts = dav.listCalendarEvents(params.url.toString(), hrefs); // Inserts/Updates data... logger.debug("Inserting/Updating events..."); HashMap<String, OEvent> cache = new HashMap<>(); for (DavCalendarEvent devt : devts) { String href = FilenameUtils.getName(devt.getPath()); //String href = devt.getPath(); if (logger.isTraceEnabled()) logger.trace("{}", ICalendarUtils.print(ICalendarUtils.getVEvent(devt.getCalendar()))); List<Integer> eventIds = eventIdsByHref.get(href); Integer eventId = (eventIds != null) ? eventIds.get(eventIds.size() - 1) : null; final ArrayList<EventInput> input = icalInput .fromICalendarFile(devt.getCalendar(), null); if (input.size() != 1) throw new WTException("iCal must contain one event"); final EventInput ei = input.get(0); if (eventId != null) { doEventDelete(con, eventId, false); } ei.event.setCalendarId(calendarId); ei.event.setHref(href); ei.event.setEtag(devt.geteTag()); doEventInputInsert(con, cache, ei); } } calDao.updateRemoteSyncById(con, calendarId, newLastSync, newSyncToken); DbUtils.commitQuietly(con); } catch (Exception ex) { DbUtils.rollbackQuietly(con); throw new WTException(ex, "Error importing iCalendar"); } } else { // Full update or partial computing hashes String newSyncToken = null; if (syncIsSupported) { // If supported, saves last sync-token issued by the server newSyncToken = dcal.getSyncToken(); } // Retrieves cards from DAV endpoint logger.debug("Querying CalDAV endpoint [{}]", params.url.toString()); List<DavCalendarEvent> devts = dav.listCalendarEvents(params.url.toString()); logger.debug("Returned {} items", devts.size()); // Handles data... try { Map<String, VEventHrefSync> syncByHref = null; if (full) { logger.debug("Cleaning up calendar [{}]", calendarId); doEventsDeleteByCalendar(con, calendarId, false); } else if (!full && !syncIsSupported) { // This hash-map is only needed when syncing using hashes EventDAO evtDao = EventDAO.getInstance(); syncByHref = evtDao.viewHrefSyncDataByCalendar(con, calendarId); } logger.debug("Processing results..."); // Define a simple map in order to check duplicates. // eg. SOGo passes same card twice :( HashSet<String> hrefs = new HashSet<>(); HashMap<String, OEvent> cache = new HashMap<>(); for (DavCalendarEvent devt : devts) { String href = PathUtils.getFileName(devt.getPath()); //String href = devt.getPath(); String etag = devt.geteTag(); if (logger.isTraceEnabled()) logger.trace("{}", ICalendarUtils.print(ICalendarUtils.getVEvent(devt.getCalendar()))); if (hrefs.contains(href)) { logger.trace("Card duplicated. Skipped! [{}]", href); continue; } boolean skip = false; Integer matchingEventId = null; if (syncByHref != null) { // Only if... (!full && !syncIsSupported) see above! //String prodId = ICalendarUtils.buildProdId(ManagerUtils.getProductName()); //String hash = DigestUtils.md5Hex(new ICalendarOutput(prodId, true).write(devt.getCalendar())); String hash = DigestUtils .md5Hex(ICalendarUtils.getVEvent(devt.getCalendar()).toString()); VEventHrefSync hrefSync = syncByHref.remove(href); if (hrefSync != null) { // Href found -> maybe updated item if (!StringUtils.equals(hrefSync.getEtag(), hash)) { matchingEventId = hrefSync.getEventId(); etag = hash; logger.trace("Event updated [{}, {}]", href, hash); } else { skip = true; logger.trace("Event not modified [{}, {}]", href, hash); } } else { // Href not found -> added item logger.trace("Event newly added [{}]", href); etag = hash; } } if (!skip) { final ArrayList<EventInput> input = icalInput .fromICalendarFile(devt.getCalendar(), null); if (input.size() != 1) throw new WTException("iCal must contain one event"); final EventInput ei = input.get(0); ei.event.setCalendarId(calendarId); ei.event.setHref(href); ei.event.setEtag(etag); if (matchingEventId == null) { doEventInputInsert(con, cache, ei); } else { ei.event.setEventId(matchingEventId); boolean updated = doEventInputUpdate(con, cache, ei); if (!updated) throw new WTException("Event not found [{}]", ei.event.getEventId()); } } hrefs.add(href); // Marks as processed! } if (syncByHref != null) { // Only if... (!full && !syncIsSupported) see above! // Remaining hrefs -> deleted items for (VEventHrefSync hrefSync : syncByHref.values()) { logger.trace("Event deleted [{}]", hrefSync.getHref()); doEventDelete(con, hrefSync.getEventId(), false); } } calDao.updateRemoteSyncById(con, calendarId, newLastSync, newSyncToken); DbUtils.commitQuietly(con); } catch (Exception ex) { DbUtils.rollbackQuietly(con); throw new WTException(ex, "Error importing iCalendar"); } } } catch (DavException ex) { throw new WTException(ex, "CalDAV error"); } } } catch (SQLException | DAOException ex) { throw wrapException(ex); } finally { DbUtils.closeQuietly(con); pendingRemoteCalendarSyncs.remove(PENDING_KEY); } }
From source file:org.apache.ambari.server.controller.metrics.ganglia.GangliaPropertyProviderTest.java
@Test public void testPopulateResources() throws Exception { TestStreamProvider streamProvider = new TestStreamProvider("temporal_ganglia_data.txt"); TestGangliaHostProvider hostProvider = new TestGangliaHostProvider(); Map<String, Map<String, PropertyInfo>> gangliaPropertyIds = PropertyHelper .getMetricPropertyIds(Resource.Type.HostComponent); GangliaPropertyProvider propertyProvider = new GangliaHostComponentPropertyProvider(gangliaPropertyIds, streamProvider, configuration, hostProvider, CLUSTER_NAME_PROPERTY_ID, HOST_NAME_PROPERTY_ID, COMPONENT_NAME_PROPERTY_ID); // namenode//w w w .j ava 2s . c o m Resource resource = new ResourceImpl(Resource.Type.HostComponent); resource.setProperty(HOST_NAME_PROPERTY_ID, "domU-12-31-39-0E-34-E1.compute-1.internal"); resource.setProperty(COMPONENT_NAME_PROPERTY_ID, "DATANODE"); // only ask for one property Map<String, TemporalInfo> temporalInfoMap = new HashMap<String, TemporalInfo>(); temporalInfoMap.put(PROPERTY_ID, new TemporalInfoImpl(10L, 20L, 1L)); Request request = PropertyHelper.getReadRequest(Collections.singleton(PROPERTY_ID), temporalInfoMap); Assert.assertEquals(1, propertyProvider.populateResources(Collections.singleton(resource), request, null).size()); String expected = (configuration.isGangliaSSL() ? "https" : "http") + "://domU-12-31-39-0E-34-E1.compute-1.internal/cgi-bin/rrd.py?c=HDPDataNode%2CHDPSlaves&h=domU-12-31-39-0E-34-E1.compute-1.internal&m=jvm.metrics.gcCount&s=10&e=20&r=1"; Assert.assertEquals(expected, streamProvider.getLastSpec()); Assert.assertEquals(3, PropertyHelper.getProperties(resource).size()); Assert.assertNotNull(resource.getPropertyValue(PROPERTY_ID)); // tasktracker resource = new ResourceImpl(Resource.Type.HostComponent); resource.setProperty(HOST_NAME_PROPERTY_ID, "domU-12-31-39-0E-34-E1.compute-1.internal"); resource.setProperty(COMPONENT_NAME_PROPERTY_ID, "TASKTRACKER"); // only ask for one property temporalInfoMap = new HashMap<String, TemporalInfo>(); Set<String> properties = new HashSet<String>(); String shuffle_exceptions_caught = PropertyHelper.getPropertyId("metrics/mapred/shuffleOutput", "shuffle_exceptions_caught"); String shuffle_failed_outputs = PropertyHelper.getPropertyId("metrics/mapred/shuffleOutput", "shuffle_failed_outputs"); String shuffle_output_bytes = PropertyHelper.getPropertyId("metrics/mapred/shuffleOutput", "shuffle_output_bytes"); String shuffle_success_outputs = PropertyHelper.getPropertyId("metrics/mapred/shuffleOutput", "shuffle_success_outputs"); properties.add(shuffle_exceptions_caught); properties.add(shuffle_failed_outputs); properties.add(shuffle_output_bytes); properties.add(shuffle_success_outputs); request = PropertyHelper.getReadRequest(properties, temporalInfoMap); temporalInfoMap.put(shuffle_exceptions_caught, new TemporalInfoImpl(10L, 20L, 1L)); temporalInfoMap.put(shuffle_failed_outputs, new TemporalInfoImpl(10L, 20L, 1L)); temporalInfoMap.put(shuffle_output_bytes, new TemporalInfoImpl(10L, 20L, 1L)); temporalInfoMap.put(shuffle_success_outputs, new TemporalInfoImpl(10L, 20L, 1L)); Assert.assertEquals(1, propertyProvider.populateResources(Collections.singleton(resource), request, null).size()); List<String> metricsRegexes = new ArrayList<String>(); metricsRegexes.add("metrics/mapred/shuffleOutput/shuffle_exceptions_caught"); metricsRegexes.add("metrics/mapred/shuffleOutput/shuffle_failed_outputs"); metricsRegexes.add("metrics/mapred/shuffleOutput/shuffle_output_bytes"); metricsRegexes.add("metrics/mapred/shuffleOutput/shuffle_success_outputs"); String metricsList = getMetricsRegexes(metricsRegexes, gangliaPropertyIds, "TASKTRACKER"); URIBuilder expectedUri = new URIBuilder(); expectedUri.setScheme((configuration.isGangliaSSL() ? "https" : "http")); expectedUri.setHost("domU-12-31-39-0E-34-E1.compute-1.internal"); expectedUri.setPath("/cgi-bin/rrd.py"); expectedUri.setParameter("c", "HDPTaskTracker,HDPSlaves"); expectedUri.setParameter("h", "domU-12-31-39-0E-34-E1.compute-1.internal"); expectedUri.setParameter("m", metricsList); expectedUri.setParameter("s", "10"); expectedUri.setParameter("e", "20"); expectedUri.setParameter("r", "1"); URIBuilder actualUri = new URIBuilder(streamProvider.getLastSpec()); Assert.assertEquals(expectedUri.getScheme(), actualUri.getScheme()); Assert.assertEquals(expectedUri.getHost(), actualUri.getHost()); Assert.assertEquals(expectedUri.getPath(), actualUri.getPath()); Assert.assertTrue(isUrlParamsEquals(actualUri, expectedUri)); Assert.assertEquals(6, PropertyHelper.getProperties(resource).size()); Assert.assertNotNull(resource.getPropertyValue(shuffle_exceptions_caught)); Number[][] dataPoints = (Number[][]) resource.getPropertyValue(shuffle_exceptions_caught); Assert.assertEquals(106, dataPoints.length); for (int i = 0; i < dataPoints.length; ++i) { Assert.assertEquals(i >= 10 && i < 20 ? 7 : 0.0, dataPoints[i][0]); Assert.assertEquals(360 * i + 1358434800, dataPoints[i][1]); } Assert.assertNotNull(resource.getPropertyValue(shuffle_failed_outputs)); Assert.assertNotNull(resource.getPropertyValue(shuffle_output_bytes)); Assert.assertNotNull(resource.getPropertyValue(shuffle_success_outputs)); }
From source file:org.apache.ambari.server.controller.metrics.ganglia.GangliaPropertyProviderTest.java
@Test public void testPopulateResources__LargeNumberOfHostResources() throws Exception { TestStreamProvider streamProvider = new TestStreamProvider("temporal_ganglia_data.txt"); TestGangliaHostProvider hostProvider = new TestGangliaHostProvider(); GangliaPropertyProvider propertyProvider = new GangliaHostPropertyProvider( PropertyHelper.getMetricPropertyIds(Resource.Type.Host), streamProvider, configuration, hostProvider, CLUSTER_NAME_PROPERTY_ID, HOST_NAME_PROPERTY_ID); Set<Resource> resources = new HashSet<Resource>(); StringBuilder hostsList = new StringBuilder(); for (int i = 0; i < 150; ++i) { Resource resource = new ResourceImpl(Resource.Type.Host); resource.setProperty(HOST_NAME_PROPERTY_ID, "host" + i); resources.add(resource);/*from w w w . j a v a2 s . com*/ if (hostsList.length() != 0) hostsList.append("," + "host" + i); else hostsList.append("host" + i); } // only ask for one property Map<String, TemporalInfo> temporalInfoMap = new HashMap<String, TemporalInfo>(); temporalInfoMap.put(PROPERTY_ID, new TemporalInfoImpl(10L, 20L, 1L)); Request request = PropertyHelper.getReadRequest(Collections.singleton(PROPERTY_ID), temporalInfoMap); Assert.assertEquals(150, propertyProvider.populateResources(resources, request, null).size()); URIBuilder expectedUri = new URIBuilder(); expectedUri.setScheme((configuration.isGangliaSSL() ? "https" : "http")); expectedUri.setHost("domU-12-31-39-0E-34-E1.compute-1.internal"); expectedUri.setPath("/cgi-bin/rrd.py"); expectedUri.setParameter("c", "HDPJobTracker,HDPHBaseMaster,HDPResourceManager,HDPFlumeServer,HDPSlaves,HDPHistoryServer,HDPJournalNode,HDPTaskTracker,HDPHBaseRegionServer,HDPNameNode"); expectedUri.setParameter("h", hostsList.toString()); expectedUri.setParameter("m", "jvm.metrics.gcCount"); expectedUri.setParameter("s", "10"); expectedUri.setParameter("e", "20"); expectedUri.setParameter("r", "1"); URIBuilder actualUri = new URIBuilder(streamProvider.getLastSpec()); Assert.assertEquals(expectedUri.getScheme(), actualUri.getScheme()); Assert.assertEquals(expectedUri.getHost(), actualUri.getHost()); Assert.assertEquals(expectedUri.getPath(), actualUri.getPath()); Assert.assertTrue(isUrlParamsEquals(actualUri, expectedUri)); }
From source file:org.apache.ambari.server.controller.metrics.ganglia.GangliaPropertyProviderTest.java
@Test public void testPopulateResources_params() throws Exception { TestStreamProvider streamProvider = new TestStreamProvider("flume_ganglia_data.txt"); TestGangliaHostProvider hostProvider = new TestGangliaHostProvider(); Map<String, Map<String, PropertyInfo>> gangliaPropertyIds = PropertyHelper .getMetricPropertyIds(Resource.Type.HostComponent); GangliaPropertyProvider propertyProvider = new GangliaHostComponentPropertyProvider(gangliaPropertyIds, streamProvider, configuration, hostProvider, CLUSTER_NAME_PROPERTY_ID, HOST_NAME_PROPERTY_ID, COMPONENT_NAME_PROPERTY_ID); // flume/*from ww w . j av a2 s.c om*/ Resource resource = new ResourceImpl(Resource.Type.HostComponent); resource.setProperty(HOST_NAME_PROPERTY_ID, "ip-10-39-113-33.ec2.internal"); resource.setProperty(COMPONENT_NAME_PROPERTY_ID, "FLUME_HANDLER"); // only ask for one property Map<String, TemporalInfo> temporalInfoMap = new HashMap<String, TemporalInfo>(); temporalInfoMap.put(FLUME_CHANNEL_CAPACITY_PROPERTY, new TemporalInfoImpl(10L, 20L, 1L)); Request request = PropertyHelper.getReadRequest(Collections.singleton(FLUME_CHANNEL_CAPACITY_PROPERTY), temporalInfoMap); Assert.assertEquals(1, propertyProvider.populateResources(Collections.singleton(resource), request, null).size()); List<String> metricsRegexes = new ArrayList<String>(); metricsRegexes.add(FLUME_CHANNEL_CAPACITY_PROPERTY); String metricsList = getMetricsRegexes(metricsRegexes, gangliaPropertyIds, "FLUME_HANDLER"); URIBuilder expectedUri = new URIBuilder(); expectedUri.setScheme((configuration.isGangliaSSL() ? "https" : "http")); expectedUri.setHost("domU-12-31-39-0E-34-E1.compute-1.internal"); expectedUri.setPath("/cgi-bin/rrd.py"); expectedUri.setParameter("c", "HDPFlumeServer,HDPSlaves"); expectedUri.setParameter("h", "ip-10-39-113-33.ec2.internal"); expectedUri.setParameter("m", metricsList); expectedUri.setParameter("s", "10"); expectedUri.setParameter("e", "20"); expectedUri.setParameter("r", "1"); URIBuilder actualUri = new URIBuilder(streamProvider.getLastSpec()); Assert.assertEquals(expectedUri.getScheme(), actualUri.getScheme()); Assert.assertEquals(expectedUri.getHost(), actualUri.getHost()); Assert.assertEquals(expectedUri.getPath(), actualUri.getPath()); Assert.assertTrue(isUrlParamsEquals(actualUri, expectedUri)); Assert.assertEquals(3, PropertyHelper.getProperties(resource).size()); Assert.assertNotNull(resource.getPropertyValue(FLUME_CHANNEL_CAPACITY_PROPERTY)); }