List of usage examples for org.apache.commons.logging Log warn
void warn(Object message);
From source file:edu.stanford.muse.util.Util.java
public static void print_exception(String message, Throwable t, Log log) { String trace = stackTrace(t); String s = message + "\n" + t.toString() + "\n" + trace; if (log != null) log.warn(s); System.err.println(s);/*from w w w. j a v a 2s . co m*/ }
From source file:com.flexive.tests.browser.AbstractBackendBrowserTest.java
/** * prints the current FXMessages to the log * @param LOG the Log to print to/*from www . java2 s .c o m*/ */ private void printFXMessages(Log LOG) { String tmpHTML = selenium.getHTMLSource("<div ", "<tbody>", "</tbody>", "<tr>", "<td>", "</td>"); int begin = tmpHTML.indexOf("<div id=\"fxMessages\""); if (begin > 0) { int end; begin = tmpHTML.indexOf("<tbody>", begin); end = tmpHTML.indexOf("</tbody>", begin); if (begin * end > 1) { String tmpTbody = tmpHTML.substring(begin, end); String[] trS = tmpTbody.split("<tr>"); String[] tdS; String tmpMSG; for (String tr : trS) { tdS = tr.split("<td>"); if (tdS.length == 3) { tmpMSG = tdS[2].substring(0, tdS[2].lastIndexOf("</td>")); if (tdS[1].indexOf("error") > 0) { LOG.error(tmpMSG); throw new RuntimeException(tmpMSG); } else { LOG.warn(tmpMSG); } } } } } }
From source file:ch.ethz.epics.export.GoogleImageSitemap.java
public static String getImageSitemapEntry(XPort xport, CumulusTools ct, Log log, String globalId, Document doc, String lang) throws JDOMException, IOException { Element recordNode = getSingleNode(doc, "//epics:record"); String recName = recordNode.getAttributeValue("recordName", XPort.nsEpics); log.debug("- image sitemap building for internal id='" + recName + "'"); Document sitemap = new Document(); // urlset sitemap root Element urlset = new Element("urlset", nsSitemap); urlset.addNamespaceDeclaration(nsImageSitemap); sitemap.setRootElement(urlset);//from ww w . ja va 2 s . c om Element url = new Element("url", nsSitemap); urlset.addContent(url); // <loc> tag specifies the URL for an original image on your site Element c = new Element("loc", nsSitemap); String baseUrl = (String) xport.getParam("index.baseUrl"); c.setText(baseUrl + "images/" + globalId + ".jpg"); url.addContent(c); /** * The <priority> value specifies the importance of a particular image relative to * other images on the same site */ c = new Element("priority", nsSitemap); c.setText("0.5"); url.addContent(c); /** * The <lastmod> value identifies the time that the content at the URL was last modified. We recommend * you set this value to the most recent time that the image last changed. This information enables * crawlers to avoid recrawling content that has not changed. */ c = new Element("lastmod", nsSitemap); // uses e-pics record modification date Element recMod = getSingleNode(doc, "//epics:field[@epics:id='{af4b2e02-5f6a-11d2-8f20-0000c0e166dc}']"); String strDate = recMod.getChildTextNormalize("content", XPort.nsEpics); SimpleDateFormat df = ct.getDefaultDateFormat(); java.util.Date dtRecMod; try { dtRecMod = df.parse(strDate); SimpleDateFormat w3c = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'"); strDate = w3c.format(dtRecMod); c.setText(strDate); url.addContent(c); } catch (ParseException e1) { // cannot parse date: log warning, leave away lastmod log.warn("can't parse record modification date for " + globalId); } /** * The <changefreq> value indicates how frequently the content at a particular URL is likely to * change. We recommend you set this value to indicate how frequently the image changes. */ c = new Element("changefreq", nsSitemap); c.setText("monthly"); url.addContent(c); /** * The <expires> tag identifies the time that the content expires. The value of the <expires> tag * should be a timestamp in W3C DATETIME format. */ // e-pics: add 2 years to now c = new Element("expires", nsSitemap); SimpleDateFormat w3c = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'"); Calendar cNow = Calendar.getInstance(); cNow.add(Calendar.YEAR, 2); Date dt = cNow.getTime(); strDate = w3c.format(dt); c.setText(strDate); url.addContent(c); // image:image Element imageTag = new Element("image", nsImageSitemap); url.addContent(imageTag); // image:landing_page Element cLandingPage = new Element("landing_page", nsImageSitemap); imageTag.addContent(cLandingPage); /** * The <image:loc> tag identifies the URL where the user will be directed when clicking on the image * from Google search results. Please note that this value must be properly encoded. */ c = new Element("loc", nsImageSitemap); baseUrl = (String) xport.getParam("index.baseUrl"); c.setText(baseUrl + globalId + ".html"); cLandingPage.addContent(c); /** * The <image:language> tag identifies the language for each landing page defined by <image:landing_page> */ // e-pics: primarily german content c = new Element("language", nsImageSitemap); c.setText(lang); cLandingPage.addContent(c); /** * The <image:title> tag specifies the title of the image. There should be a maximum of one title * per landing page. */ Element e = getSingleNode(doc, "//epics:field[@epics:id='{af4b2e3d-5f6a-11d2-8f20-0000c0e166dc}']"); String title = e.getChildText("content", XPort.nsEpics); if (title != null && title.length() > 0) { c = new Element("title", nsImageSitemap); c.setText(title); cLandingPage.addContent(c); } /** * The <image:caption> tag's value provides context for an image. Please note that this value must be XMLencoded. * There should be a maximum of one caption per landing page. */ e = getSingleNode(doc, "//epics:field[@epics:id='{af4b2e34-5f6a-11d2-8f20-0000c0e166dc}']"); String caption = e.getChildText("content", XPort.nsEpics); if (caption != null && caption.length() > 0) { c = new Element("caption", nsImageSitemap); c.setText(caption); cLandingPage.addContent(c); } /** * The <image:category> tag's value specifies one or more groups, subjects or categories that * describe the image. Multiple categories should be included in separate category tags. * Comma-separated values within a single category tag will be considered one single category. */ //ArrayList<String> keywords = new ArrayList<String>(); HashSet<String> keywords = new HashSet<String>(); XPath xpath = XPath.newInstance("//epics:category"); xpath.addNamespace(XPort.nsEpics); List catNodes = xpath.selectNodes(doc); Iterator it = catNodes.iterator(); String locationName = ""; while (it.hasNext()) { e = (Element) it.next(); String catName = e.getTextNormalize(); log.debug("catName: " + catName); // KJ/20081203: use all keywords /* last keyword: String kw = catName.replaceAll(".* >? (.*)", "$1"); log.debug("got keyword: "+kw); if (kw != null && kw.length() > 0) { keywords.addAll(Arrays.asList(kw.split(", "))); } */ /* if (catName != null && catName.length() > 0) { keywords.addAll(Arrays.asList(catName.split("> "))); log.debug("got keywords: "+keywords); } */ // add keywords with duplicates removed if (catName != null && catName.length() > 0) { List<String> kw = Arrays.asList(catName.split(">")); for (String k : kw) { if (!keywords.contains(k) && k.trim().length() > 1) { log.debug("got keyword: " + k); keywords.add(k.trim()); } } } String nCatName = catName.replaceAll("\\s*>", ", "); if (nCatName != null && nCatName.length() > 0) { // KJ/20081203: don't add categories - use keywords instead //c = new Element("category", nsImageSitemap); //c.setText(nCatName); //cLandingPage.addContent(c); // E-Pics ETHBIB.Bildarchiv *spezial* if (catName.contains("Politische Geographie")) { log.debug("found location = " + catName); catName = catName.replaceAll(", Kanton > ", ", "); catName = catName.replaceAll(", Stadt > ", ", "); locationName = catName.replaceAll(".*?Politische Geographie\\s*>*(.*)", "$1"); log.debug("reduced location to = " + locationName); locationName = locationName.replaceAll("\\s*>", ","); log.debug("reduced location to = " + locationName); } } /* if (nCatName.contains("Sachkatalog")) { String kwlist = catName.replaceAll(".*?Sachkatalog\\s*>*(.*)", "$1"); if (kwlist != null && kwlist.length() > 0) { keywords.addAll(Arrays.asList(kwlist.split(", "))); } } */ } /** * The <image:keyword> tag contains a single keyword that describes an image. By properly tagging * images, you will help us to rank them in the Google Image Index. Please provide keywords that * are as specific and descriptive as possible. Broad keywords may or may not be used in indexing. * Keywords should be included in separate keyword tags, and comma-separated values within a single * keyword tag will be considered one single keyword. */ it = keywords.iterator(); while (it.hasNext()) { String kw = (String) it.next(); c = new Element("keyword", nsImageSitemap); c.setText(kw); cLandingPage.addContent(c); } /** * The <image:family_friendly> tag's value indicates whether the image only contains content that * is suitable for children. Acceptable values for this tag are yes and no. Please use reasonable * judgment when determining values for this tag. One way to define family-friendly is whether * the image could appear in a G-rated movie. */ c = new Element("family_friendly", nsImageSitemap); //TODO //e = (Element)xpath.selectSingleNode("//epics:field[@epics:id='{af4b2e34-5f6a-11d2-8f20-0000c0e166dc}']"); //String familyFriendly = e.getChildText("epics:content"); c.setText("yes"); imageTag.addContent(c); /** * The <image:geo_location> tag is used to specify a geographical location. This can be a string * the form of an address, city name, or latitude and longitude pair. Please note that this value * must be XML-encoded. */ c = new Element("geo_location", nsImageSitemap); if (locationName != null) { c.setText(locationName); imageTag.addContent(c); } /** * The <image:geo_location> tag is used to specify a geographical location. This can be a string * the form of an address, city name, or latitude and longitude pair. Please note that this value * must be XML-encoded. */ // E-Pics: Creative Commons Namensnennung, Non-commercial, no derivatives c = new Element("license", nsImageSitemap); c.setText("http://creativecommons.org/licenses/by-nc-nd/2.5/ch/"); imageTag.addContent(c); /** * The <image:quality> tag's value specifies the quality of the image relative to other images. * This information may be used to rank images from the same site relative to one another on * search result pages. Unlike <priority>, it is not used to prioritize images indexed by Google. */ // E-Pics: not used //c = new Element("quality", nsImageSitemap); //c.setText("1.0"); //imageTag.addContent(c); /** * The <image:publication_date> tag identifies the original publication date of the image in YYYY-MM-DD format. * The value of the <image:publication_date> tag should be a timestamp in W3C DATETIME format. */ // E-Pics: EXIF Date created -or- TODO: Date field c = new Element("publication_date", nsImageSitemap); Element pubDate = getSingleNode(doc, "//epics:field[@epics:id='{af4b2e51-5f6a-11d2-8f20-0000c0e166dc}']"); if (pubDate != null) { strDate = pubDate.getChildTextNormalize("content", XPort.nsEpics); } else { // try to get date field Element dateField = getSingleNode(doc, "//epics:field[@epics:id='{132267c2-4148-4b76-b851-88409d7d2799}']"); if (dateField != null) { strDate = dateField.getChildTextNormalize("content", XPort.nsEpics); if (strDate != null && strDate.length() > 0) { if (!strDate.contains(".") && !strDate.contains("/")) { // year only strDate = "01.01." + strDate; //+" 00:00:00 CET"; } else if (strDate.contains("/") && strDate.length() > 0) { // mm/YYYY String[] dateDetails = strDate.split("\\/"); if (dateDetails.length == 2) { strDate = "01." + dateDetails[0] + "." + dateDetails[1]; //strDate += " 00:00:00 CET"; } } else { //strDate = strDate+" 00:00:00 CET"; } log.debug("converted dateonly value = " + strDate); } } } if (strDate != null && strDate.length() > 0) { df = ct.getDefaultDateFormat(); java.util.Date dtPubDate; try { dtPubDate = df.parse(strDate); w3c = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'"); strDate = w3c.format(dtPubDate); c.setText(strDate); // KJ: 20090619 - Google does not like publication dates < 1970 // therefore they are removed from the sitemap //imageTag.addContent(c); } catch (ParseException e1) { // cannot parse date: log warning, leave away lastmod log.warn("can't parse publication date for " + globalId, e1); } } /** * The <image:size> tag specifies the size of the image in pixels. The images that you make * available to Google's crawlers should be the same size as the images that you display to * users on your site. */ // E-Pics: will need size of derivative image Element mediumImage = getSingleNode(doc, "//epics:field[@epics:fieldName='Image']"); String finalSize = mediumImage.getAttributeValue("size", XPort.nsEpics); if (finalSize != null) { c = new Element("size", nsImageSitemap); c.setText(finalSize); imageTag.addContent(c); } /** * The <image:watermarking> tag's value indicates whether watermarking exists on the image. The * only valid values of this tag are yes and no. If the value is yes, then you have the option * of specifying the percentage attribute: */ // E-Pics: maximum of 5% for creative commons license c = new Element("watermarking", nsImageSitemap); c.setText("yes"); c.setAttribute("percentage", "5"); imageTag.addContent(c); // write XML fragment into string StringWriter sw = new StringWriter(); Format xmlFormatDebug = Format.getPrettyFormat().setOmitDeclaration(true); xmlFormatDebug.setEncoding("UTF-8"); XMLOutputter xmlOutput = new XMLOutputter(xmlFormatDebug); Element sitemapRoot = sitemap.getRootElement(); xmlOutput.output((Element) sitemapRoot.getContent(0), sw); String part = sw.toString(); part = part.replaceAll(" xmlns=\"http://www.google.com/schemas/sitemap/0.9\"", ""); part = part.replaceAll(" xmlns\\:image=\"http://www.google.com/schemas/sitemap/0.9\"", ""); return part; }
From source file:io.smartspaces.system.bootstrap.osgi.GeneralSmartSpacesSupportActivator.java
/** * Set up the system configuration.//from w w w . j a v a2 s. c o m * * @param containerProperties * properties for the container * @param log * the logger to use */ private void setupSystemConfiguration(Map<String, String> containerProperties, Log log) { expressionEvaluatorFactory = new SselExpressionEvaluatorFactory(); FileSystemConfigurationStorageManager fileSystemConfigurationStorageManager = new FileSystemConfigurationStorageManager(); fileSystemConfigurationStorageManager.setLog(spaceEnvironment.getLog()); fileSystemConfigurationStorageManager.setExpressionEvaluatorFactory(expressionEvaluatorFactory); fileSystemConfigurationStorageManager.setSmartspacesFilesystem(filesystem); fileSystemConfigurationStorageManager.setConfigFolder(configurationProvider.getConfigFolder()); systemConfigurationStorageManager = fileSystemConfigurationStorageManager; systemConfigurationStorageManager.startup(); containerManagedScope.addResource(systemConfigurationStorageManager); Configuration systemConfiguration = systemConfigurationStorageManager.getSystemConfiguration(); systemConfiguration.setProperties(containerProperties); systemConfiguration.setProperty(SmartSpacesEnvironment.CONFIGURATION_NAME_SMARTSPACES_VERSION, bundleContext.getProperty(CoreConfiguration.CONFIGURATION_NAME_SMARTSPACES_VERSION)); String hostAddress = getHostAddress(systemConfiguration); if (hostAddress != null) { log.info(String.format("Using container host address %s", hostAddress)); systemConfiguration.setProperty(SmartSpacesEnvironment.CONFIGURATION_NAME_HOST_ADDRESS, hostAddress); } else { log.warn("Could not determine container host address."); } systemConfiguration.setProperty(SmartSpacesEnvironment.CONFIGURATION_NAME_SYSTEM_FILESYSTEM_DIR_INSTALL, filesystem.getInstallDirectory().getAbsolutePath()); systemConfiguration.setProperty(SmartSpacesEnvironment.CONFIGURATION_NAME_SYSTEM_FILESYSTEM_DIR_DATA, filesystem.getDataDirectory().getAbsolutePath()); systemConfiguration.setProperty(SmartSpacesEnvironment.CONFIGURATION_NAME_SYSTEM_FILESYSTEM_DIR_TMP, filesystem.getTempDirectory().getAbsolutePath()); spaceEnvironment.setSystemConfiguration(systemConfiguration); }
From source file:edu.stanford.muse.util.Util.java
/** cleans up files in directory with the given suffix */ public static void deleteAllFilesWithSuffix(String dir, String suffix, Log log) throws IOException, ClassNotFoundException { if (dir == null) return;/*from w ww. j a v a 2 s. c om*/ File cache = new File(dir); if (!cache.exists()) return; // empty result File files[] = new File(dir).listFiles(new Util.MyFilenameFilter(null, suffix)); if (files != null) for (File f : files) { boolean success = f.delete(); if (log != null) { if (success) log.info("Deleted file: " + f.getName()); else log.warn("Failed to delete file: " + f.getName()); } } }
From source file:com.curl.orb.servlet.InstanceManagementServlet.java
@Override public void init() { synchronized (initializedLock) { if (!initializedLock.isFirstTime()) return; Log log = LogFactory.getLog(getClass()); // Set to environment String environment = getServletContext().getInitParameter(ENVIRONMENT); if (environment != null) { if (environment.equalsIgnoreCase(Environment.PRODUCTION.toString())) InstanceManagementServlet.environment = Environment.PRODUCTION; else if (environment.equalsIgnoreCase(Environment.TEST.toString())) InstanceManagementServlet.environment = Environment.TEST; else if (environment.equalsIgnoreCase(Environment.DEVELOPMENT.toString())) InstanceManagementServlet.environment = Environment.DEVELOPMENT; }/*from ww w. j av a2s . co m*/ log.info("Environment [" + environment + "]"); // Load ApplicationContext AbstractApplicationContext applicationContext = (ApplicationContextFactory .getInstance(getServletContext())).getApplicationContext(); log.info("Loaded ApplicationContext [" + applicationContext + "]"); // Load jars/classes files try { if (!(Environment.TEST.contain(InstanceManagementServlet.environment)) || InstanceManagementServlet.environment == null) { ClassPathLoader loader = ClassPathLoader .getInstance(getServletContext().getInitParameter(GENERATOR_FILTER)); String fileType = getServletContext().getInitParameter(GENERATOR_LOAD_FILETYPE); if (fileType == null || fileType.equalsIgnoreCase("both")) { loader.addClassProperties(getServletContext().getRealPath("/WEB-INF/lib")); loader.addClassProperties(getServletContext().getRealPath("/WEB-INF/classes")); } else if (fileType.equalsIgnoreCase("class")) { loader.addClassProperties(getServletContext().getRealPath("/WEB-INF/classes")); } else if (fileType.equalsIgnoreCase("jar")) { loader.addClassProperties(getServletContext().getRealPath("/WEB-INF/lib")); } // System.getProperty("java.class.path") log.info("Loaded java libraries to generate Curl code."); } } catch (GeneratorException e) { log.warn(e); } } }
From source file:com.amazon.carbonado.repo.replicated.ReplicatedRepository.java
@SuppressWarnings("unchecked") private <S extends Storable> void resync(ReplicationTrigger<S> replicationTrigger, Storage<S> replicaStorage, Query<S> replicaQuery, Storage<S> masterStorage, Query<S> masterQuery, ResyncCapability.Listener<? super S> listener, Throttle throttle, double desiredSpeed, Comparator comparator, Transaction replicaTxn) throws RepositoryException { final Log log = LogFactory.getLog(ReplicatedRepository.class); Cursor<S> replicaCursor = null; Cursor<S> masterCursor = null; try {/*from ww w . j a va 2 s .c o m*/ while (replicaCursor == null) { try { replicaCursor = replicaQuery.fetch(); } catch (CorruptEncodingException e) { S replicaWithKeyOnly = recoverReplicaKey(replicaStorage, e); if (!deleteCorruptEntry(replicationTrigger, replicaWithKeyOnly, e)) { // Cannot delete it, so just give up. throw e; } } } masterCursor = masterQuery.fetch(); S lastReplicaEntry = null; S replicaEntry = null; S masterEntry = null; int count = 0, txnCount = 0; while (true) { if (throttle != null) { try { // 100 millisecond clock precision throttle.throttle(desiredSpeed, 100); } catch (InterruptedException e) { throw new FetchInterruptedException(e); } } if (replicaEntry == null && replicaCursor != null) { int skippedCount = 0; while (replicaCursor.hasNext()) { try { replicaEntry = replicaCursor.next(); if (listener != null) { listener.afterLoad(replicaEntry); } if (skippedCount > 0) { if (skippedCount == 1) { log.warn("Skipped corrupt replica entry before this one: " + replicaEntry); } else { log.warn("Skipped " + skippedCount + " corrupt replica entries before this one: " + replicaEntry); } } break; } catch (CorruptEncodingException e) { // Exception forces cursor to close. Close again to be sure. replicaCursor.close(); replicaCursor = null; S replicaWithKeyOnly = recoverReplicaKey(replicaStorage, e); boolean deleted = deleteCorruptEntry(replicationTrigger, replicaWithKeyOnly, e); // Re-open (if we can) if (lastReplicaEntry == null) { break; } replicaCursor = replicaQuery.fetchAfter(lastReplicaEntry); if (deleted) { // Skip if it cannot be deleted. try { skippedCount = replicaCursor.skipNext(++skippedCount); log.info("Skipped corrupt replica entry", e); if (replicaWithKeyOnly != null) { // Try to update entry which could not be deleted. replicaEntry = replicaWithKeyOnly; if (listener != null) { listener.afterLoad(replicaEntry); } break; } } catch (FetchException e2) { log.error("Unable to skip past corrupt replica entry", e2); throw e; } } } } } if (count++ >= RESYNC_WATERMARK || txnCount >= RESYNC_BATCH_SIZE) { replicaTxn.commit(); if (replicaCursor != null) { // Cursor should auto-close after txn commit, but force // a close anyhow. Cursor is re-opened when it is // allowed to advance. replicaCursor.close(); replicaCursor = null; } count = 0; txnCount = 0; } if (masterEntry == null && masterCursor.hasNext()) { masterEntry = masterCursor.next(); } Runnable resyncTask = null; // Comparator should treat null as high. int compare = comparator.compare(replicaEntry, masterEntry); if (compare < 0) { // Bogus record exists only in replica so delete it. resyncTask = prepareResyncTask(replicationTrigger, listener, replicaEntry, null); // Allow replica to advance. if (replicaCursor == null) { replicaCursor = replicaQuery.fetchAfter(replicaEntry); } lastReplicaEntry = replicaEntry; replicaEntry = null; } else if (compare > 0) { // Replica cursor is missing an entry so copy it. resyncTask = prepareResyncTask(replicationTrigger, listener, null, masterEntry); // Allow master to advance. masterEntry = null; } else { // If compare is zero, replicaEntry and masterEntry are // either both null or both non-null. if (replicaEntry == null && masterEntry == null) { // Both cursors exhausted -- resync is complete. break; } // Both replicaEntry and masterEntry are non-null. if (!replicaEntry.equalProperties(masterEntry)) { // Replica is stale. resyncTask = prepareResyncTask(replicationTrigger, listener, replicaEntry, masterEntry); } // Entries are synchronized so allow both cursors to advance. if (replicaCursor == null) { replicaCursor = replicaQuery.fetchAfter(replicaEntry); } lastReplicaEntry = replicaEntry; replicaEntry = null; masterEntry = null; } if (resyncTask != null) { txnCount++; resyncTask.run(); } } } finally { try { if (masterCursor != null) { masterCursor.close(); } } finally { if (replicaCursor != null) { replicaCursor.close(); } } } }
From source file:com.amazon.carbonado.repo.indexed.IndexedStorage.java
@SuppressWarnings("unchecked") private void removeIndex(StorableIndex index) throws RepositoryException { Log log = LogFactory.getLog(IndexedStorage.class); if (log.isInfoEnabled()) { StringBuilder b = new StringBuilder(); b.append("Removing index on "); b.append(getStorableType().getName()); b.append(": "); try {/*from ww w . j a v a 2 s.c om*/ index.appendTo(b); } catch (java.io.IOException e) { // Not gonna happen. } log.info(b.toString()); } Class<? extends Storable> indexEntryClass = IndexEntryGenerator.getIndexAccess(index).getReferenceClass(); Storage<?> indexEntryStorage; try { indexEntryStorage = mRepository.getIndexEntryStorageFor(indexEntryClass); } catch (Exception e) { // Assume it doesn't exist. unregisterIndex(index); return; } { // Doesn't completely remove the index, but it should free up space. double desiredSpeed = mRepository.getIndexRepairThrottle(); Throttle throttle = desiredSpeed < 1.0 ? new Throttle(BUILD_THROTTLE_WINDOW) : null; if (throttle == null) { indexEntryStorage.truncate(); } else { long totalDropped = 0; while (true) { Transaction txn = mRepository.getWrappedRepository() .enterTopTransaction(IsolationLevel.READ_COMMITTED); txn.setForUpdate(true); try { Cursor<? extends Storable> cursor = indexEntryStorage.query().fetch(); if (!cursor.hasNext()) { break; } int count = 0; final long savedTotal = totalDropped; boolean anyFailure = false; try { while (count++ < BUILD_BATCH_SIZE && cursor.hasNext()) { if (cursor.next().tryDelete()) { totalDropped++; } else { anyFailure = true; } } } finally { cursor.close(); } txn.commit(); if (log.isInfoEnabled()) { log.info("Removed " + totalDropped + " index entries"); } if (anyFailure && totalDropped <= savedTotal) { log.warn("No indexes removed in last batch. " + "Aborting index removal cleanup"); break; } } catch (FetchException e) { throw e.toPersistException(); } finally { txn.exit(); } try { throttle.throttle(desiredSpeed, BUILD_THROTTLE_SLEEP_PRECISION); } catch (InterruptedException e) { throw new RepositoryException("Index removal interrupted"); } } } } unregisterIndex(index); }
From source file:de.tudarmstadt.ukp.clarin.webanno.crowdflower.NamedEntityTaskManager.java
/** * Uploads a new task2 to Crowdflower, producing all data entirely of the raw judgments file * retrieved from a task1 ID./*from w w w . j a v a2s. co m*/ * * @param template the template. * @param jobID1 the job ID. * @param documentsJCas the documents. * @param goldsJCas the gold documents. * @return Crowdflower ID as string of the new task * @throws JsonProcessingException hum? * @throws IOException hum? * @throws CrowdException hum? */ public String uploadNewNERTask2(String template, String jobID1, List<JCas> documentsJCas, List<JCas> goldsJCas) throws JsonProcessingException, IOException, CrowdException { Log LOG = LogFactory.getLog(getClass()); omittedSentences = 0; // Reader that also downloades the raw judgments for the supplied job id BufferedReader br = getReaderForRawJudgments(jobID1); String line; // JSON object mapper ObjectMapper mapper = new ObjectMapper(); ObjectWriter writer = mapper.writer(); // Used to represent task2 data that is send as JSON to crowdflower Vector<NamedEntityTask2Data> uploadData = new Vector<NamedEntityTask2Data>(); // Judgments come in as a quite exotic multiline-json, we need to parse every line of it // separately while ((line = br.readLine()) != null) { // try to process each line, omit data if an error occurs (but inform user) try { JsonNode elem = mapper.readTree(line); String text = elem.path(JSON_FIELD_DATA).path(NamedEntityTask1Data.FIELD_TEXT).getTextValue(); String state = elem.path(JSON_FIELD_STATE).getTextValue(); // omit hidden gold items if (state.equals(JSON_VALUE_HIDDEN_GOLD)) { continue; } String document = elem.path(JSON_FIELD_DATA).path(JSON_FIELD_DOCUMENT).getTextValue(); int offset = elem.path(JSON_FIELD_DATA).path(NamedEntityTask1Data.FIELD_OFFSET).getIntValue(); if (state.equals(JSON_VALUE_GOLDEN)) { // produce gold data String markertext_gold = elem.path(JSON_FIELD_DATA) .path(NamedEntityTask1Data.FIELD_MARKERTEXT_GOLD).getTextValue(); String types = elem.path(JSON_FIELD_DATA).path(NamedEntityTask1Data.FIELD_TYPES).getTextValue(); if (!types.equals(JSON_VALUE_EMPTY_ARRAY)) { // sentence has atleast one NE List<String> NEtypes = Arrays.asList(types.substring(1, types.length() - 1).split(",")); JsonNode markers = mapper.readTree(markertext_gold); if (NEtypes.size() != markers.size()) { LOG.warn( "Warning, skipping ill formated gold item in task1! (NEtypes.size() != markers.size())"); continue; } int i = 0; for (JsonNode marker : markers) { int start = marker.path(JSON_FIELD_START_MARKER).getIntValue(); int end = marker.path(JSON_FIELD_END_MARKER).getIntValue(); NamedEntityTask2Data task2_gold_datum = new NamedEntityTask2Data(text, extractSpan(text, start, end), writer.writeValueAsString(marker), String.valueOf(getFirstSpanOffset(text)), document, task2NeMap.get(NEtypes.get(i)), bogusNER2Reason); task2_gold_datum.setDocOffset(offset); uploadData.add(task2_gold_datum); i++; } } // else ignore this sentence } else // normal data entry { if (!elem.path(JSON_FIELD_RESULTS).path(JSON_FIELD_JUDGMENTS).isMissingNode()) { Map<String, Integer> votings = new HashMap<String, Integer>(); // Majority voting for each marker in all judgments for (JsonNode judgment : elem.path(JSON_FIELD_RESULTS).path(JSON_FIELD_JUDGMENTS)) { if (!judgment.path(JSON_FIELD_DATA).path(NamedEntityTask1Data.FIELD_MARKERTEXT) .isMissingNode()) { String markertext = judgment.path(JSON_FIELD_DATA) .path(NamedEntityTask1Data.FIELD_MARKERTEXT).getTextValue(); JsonNode markers = mapper.readTree(markertext); // iterate over votes for (JsonNode marker : markers) { String voteText = writer.writeValueAsString(marker); // first case: add entry for this voting position if (!votings.containsKey(voteText)) { votings.put(voteText, 1); } // second case: increment voting else { votings.put(voteText, votings.get(voteText) + 1); } } } else { LOG.warn( "Warning, missing path in JSON result file from crowdflower: results/judgments"); } } // Consider any marked span which has at least two votes. Bogus spans can still be filtered out by task2 int votes_needed = 2; List<String> majorityMarkers = new ArrayList<String>(); for (String vote : votings.keySet()) { if (votings.get(vote) >= votes_needed) { majorityMarkers.add(vote); } } // process majority markers for (String strMarker : majorityMarkers) { if (!strMarker.equals(JSON_VALUE_NONE1) && !strMarker.equals(JSON_VALUE_NONE2)) { JsonNode marker = mapper.readTree(strMarker); int start = marker.path(JSON_FIELD_START_MARKER).getIntValue(); int end = marker.path(JSON_FIELD_END_MARKER).getIntValue(); NamedEntityTask2Data task2_datum = new NamedEntityTask2Data(text, extractSpan(text, start, end), strMarker, String.valueOf(getFirstSpanOffset(text)), document); task2_datum.setDocOffset(offset); uploadData.add(task2_datum); } } } else { LOG.warn("Warning, missing path in JSON result file from crowdflower: data/markertext"); } } } catch (Exception e) { omittedSentences++; LOG.warn("Warning, omitted a sentence from task2 upload because of an error in processing it: " + e.getMessage()); } } LOG.info("Data generation complete. Creating new Job for Ner task 2."); CrowdJob job = createJob(template); setAllowedCountries(job); crowdclient.updateAllowedCountries(job); LOG.info("Done, new job id is: " + job.getId() + ". Now generating data for NER task 2"); crowdclient.uploadData(job, uploadData); LOG.info("Done uploading data to task2 #" + job.getId() + "."); return job.getId(); }
From source file:com.amazon.carbonado.repo.indexed.ManagedIndex.java
/** * Build the entire index, repairing as it goes. * * @param repo used to enter transactions */// www . j a v a2s .co m void buildIndex(double desiredSpeed) throws RepositoryException { final MergeSortBuffer buffer; final Comparator c; final Log log = LogFactory.getLog(IndexedStorage.class); final Query<S> masterQuery; { // Need to explicitly order master query by primary key in order // for fetchAfter to work correctly in case corrupt records are // encountered. masterQuery = mMasterStorage.query().orderBy(naturalOrdering(mMasterStorage.getStorableType())); } // Quick check to see if any records exist in master. { Transaction txn = mRepository.enterTopTransaction(IsolationLevel.READ_COMMITTED); try { if (!masterQuery.exists()) { if (mIndexEntryStorage.query().exists()) { txn.exit(); mIndexEntryStorage.truncate(); } return; } } finally { txn.exit(); } } // Enter top transaction with isolation level of none to make sure // preload operation does not run in a long nested transaction. Transaction txn = mRepository.enterTopTransaction(IsolationLevel.NONE); try { Cursor<S> cursor = masterQuery.fetch(); try { if (log.isInfoEnabled()) { StringBuilder b = new StringBuilder(); b.append("Preparing index on "); b.append(mMasterStorage.getStorableType().getName()); b.append(": "); try { mIndex.appendTo(b); } catch (java.io.IOException e) { // Not gonna happen. } log.info(b.toString()); } // Preload and sort all index entries for improved performance. buffer = new MergeSortBuffer(mIndexEntryStorage, null, BUILD_SORT_BUFFER_SIZE); c = getComparator(); buffer.prepare(c); long nextReportTime = System.currentTimeMillis() + BUILD_INFO_DELAY_MILLIS; // These variables are used when corrupt records are encountered. S lastUserStorable = null; int skippedCount = 0; while (cursor.hasNext()) { S userStorable; try { userStorable = cursor.next(); skippedCount = 0; } catch (CorruptEncodingException e) { log.warn("Omitting corrupt record from index: " + e.toString()); // Exception forces cursor to close. Close again to be sure. cursor.close(); if (lastUserStorable == null) { cursor = masterQuery.fetch(); } else { cursor = masterQuery.fetchAfter(lastUserStorable); } cursor.skipNext(++skippedCount); continue; } buffer.add(makeIndexEntry(userStorable)); if (log.isInfoEnabled()) { long now = System.currentTimeMillis(); if (now >= nextReportTime) { log.info("Prepared " + buffer.size() + " index entries"); nextReportTime = now + BUILD_INFO_DELAY_MILLIS; } } lastUserStorable = userStorable; } // No need to commit transaction because no changes should have been made. } finally { cursor.close(); } } finally { txn.exit(); } // This is not expected to take long, since MergeSortBuffer sorts as // needed. This just finishes off what was not written to a file. buffer.sort(); if (isUnique()) { // If index is unique, scan buffer and check for duplicates // _before_ inserting index entries. If there are duplicates, // fail, since unique index cannot be built. if (log.isInfoEnabled()) { log.info("Verifying index"); } Object last = null; for (Object obj : buffer) { if (last != null) { if (c.compare(last, obj) == 0) { buffer.close(); throw new UniqueConstraintException("Cannot build unique index because duplicates exist: " + this + ", " + last + " == " + obj); } } last = obj; } } final int bufferSize = buffer.size(); if (log.isInfoEnabled()) { log.info("Begin build of " + bufferSize + " index entries"); } // Need this index entry query for deleting bogus entries. final Query indexEntryQuery = mIndexEntryStorage.query() .orderBy(naturalOrdering(mIndexEntryStorage.getStorableType())); Throttle throttle = desiredSpeed < 1.0 ? new Throttle(BUILD_THROTTLE_WINDOW) : null; long totalInserted = 0; long totalUpdated = 0; long totalDeleted = 0; long totalProgress = 0; txn = enterBuildTxn(); try { Cursor<? extends Storable> indexEntryCursor = indexEntryQuery.fetch(); Storable existingIndexEntry = null; if (!indexEntryCursor.hasNext()) { indexEntryCursor.close(); // Don't try opening again. indexEntryCursor = null; } boolean retry = false; Storable indexEntry = null; Storable lastIndexEntry = null; long nextReportTime = System.currentTimeMillis() + BUILD_INFO_DELAY_MILLIS; Iterator it = buffer.iterator(); bufferIterate: while (true) { if (!retry) { Object obj; if (it.hasNext()) { obj = it.next(); } else if (indexEntryCursor != null && indexEntryCursor.hasNext()) { obj = null; } else { break; } indexEntry = (Storable) obj; } try { if (indexEntry != null) { if (indexEntry.tryInsert()) { totalInserted++; } else { // Couldn't insert because an index entry already exists. Storable existing = indexEntry.copy(); boolean doUpdate = false; if (!existing.tryLoad()) { doUpdate = true; } else if (!existing.equalProperties(indexEntry)) { // If only the version differs, leave existing entry alone. indexEntry.copyVersionProperty(existing); doUpdate = !existing.equalProperties(indexEntry); } if (doUpdate) { indexEntry.tryDelete(); indexEntry.tryInsert(); totalUpdated++; } } } if (indexEntryCursor != null) while (true) { if (existingIndexEntry == null) { if (indexEntryCursor.hasNext()) { existingIndexEntry = indexEntryCursor.next(); } else { indexEntryCursor.close(); // Don't try opening again. indexEntryCursor = null; break; } } int compare = c.compare(existingIndexEntry, indexEntry); if (compare == 0) { // Existing entry cursor matches so allow cursor to advance. existingIndexEntry = null; break; } else if (compare > 0) { // Existing index entry is ahead so check later. break; } else { // Existing index entry might be bogus. Check again // in case master record changed. doDelete: { S master = mMasterStorage.prepare(); copyToMasterPrimaryKey(existingIndexEntry, master); if (master.tryLoad()) { Storable temp = makeIndexEntry(master); existingIndexEntry.copyVersionProperty(temp); if (existingIndexEntry.equalProperties(temp)) { break doDelete; } } existingIndexEntry.tryDelete(); totalDeleted++; if (totalDeleted % BUILD_BATCH_SIZE == 0) { txn.commit(); txn.exit(); nextReportTime = logProgress(nextReportTime, log, totalProgress, bufferSize, totalInserted, totalUpdated, totalDeleted); txn = enterBuildTxn(); indexEntryCursor.close(); indexEntryCursor = indexEntryQuery.fetchAfter(existingIndexEntry); if (!indexEntryCursor.hasNext()) { indexEntryCursor.close(); // Don't try opening again. indexEntryCursor = null; break; } } } existingIndexEntry = null; throttle(throttle, desiredSpeed); } } if (indexEntry != null) { totalProgress++; } lastIndexEntry = indexEntry; retry = false; } catch (RepositoryException e) { if (e instanceof FetchTimeoutException || e instanceof PersistTimeoutException) { log.warn("Lock conflict during index repair; will retry: " + indexEntry + ", " + e); // This re-uses the last index entry to repair and forces // the current transaction to commit. retry = true; } else { throw e; } } if (retry || (totalProgress % BUILD_BATCH_SIZE == 0)) { txn.commit(); txn.exit(); nextReportTime = logProgress(nextReportTime, log, totalProgress, bufferSize, totalInserted, totalUpdated, totalDeleted); txn = enterBuildTxn(); if (indexEntryCursor != null) { indexEntryCursor.close(); existingIndexEntry = null; if (indexEntry == null || lastIndexEntry == null) { indexEntryCursor = indexEntryQuery.fetch(); } else if (!retry) { indexEntryCursor = indexEntryQuery.fetchAfter(indexEntry); } else { // Re-fetch starting at the same spot. indexEntryCursor = indexEntryQuery.fetchAfter(lastIndexEntry); } } } throttle(throttle, desiredSpeed); } txn.commit(); } finally { txn.exit(); buffer.close(); } if (log.isInfoEnabled()) { log.info("Finished building " + totalProgress + " index entries " + progressSubMessgage(totalInserted, totalUpdated, totalDeleted)); } }