List of usage examples for java.util.regex Pattern pattern
String pattern
To view the source code for java.util.regex Pattern pattern.
Click Source Link
From source file:com.rapidminer.tools.Tools.java
/** * Replace quote chars in-quote characters by escapeChar+quotingChar * * Example: seperatorPatern = ',' , quotingChar = '"' , escapeCahr = '\\' * * line = '"Charles says://from w w w . ja va2 s . c o m * "Some people never go crazy, What truly horrible lives they must live"", 1968, " * US"' return = '"Charles says: * \"Some people never go crazy, What truly horrible lives they must live\"", "1968", "US"' */ public static String escapeQuoteCharsInQuotes(String line, Pattern separatorPattern, char quotingChar, char escapeChar, boolean showWarning) { // first remember quoteChar positions which should be escaped: char lastChar = '0'; boolean openedQuote = false; List<Integer> rememberQuotePosition = new LinkedList<>(); for (int i = 0; i < line.length(); i++) { if (lastChar == quotingChar) { if (openedQuote) { boolean matches = Pattern.matches(separatorPattern.pattern() + ".*", line.substring(i)); if (matches) { openedQuote = false; } else { rememberQuotePosition.add(i - 1); } } else { openedQuote = true; } } lastChar = line.charAt(i); } if (openedQuote && lastChar == quotingChar) { openedQuote = false; } // print warning if (showWarning && !rememberQuotePosition.isEmpty()) { StringBuilder positions = new StringBuilder(); int j = 1; for (int i = 0; i < rememberQuotePosition.size(); i++) { if (j % 10 == 0) { positions.append("\n"); } positions.append(rememberQuotePosition.get(i)); if (i + 1 < rememberQuotePosition.size()) { positions.append(", "); } j++; } String lineBeginning = line; if (line.length() > 20) { lineBeginning = line.substring(0, 20); } String warning = "While reading the line starting with \n\n\t" + lineBeginning + " ...\n\n" + ",an unescaped quote character was substituted by an escaped quote at the position(s) " + positions.toString() + ". " + "In particular der character '" + Character.toString(lastChar) + "' was replaced by '" + Character.toString(escapeChar) + Character.toString(lastChar) + "."; LogService.getRoot().log(Level.WARNING, warning); } // then build new line: if (!rememberQuotePosition.isEmpty()) { String newLine = ""; int pos = rememberQuotePosition.remove(0); int i = 0; for (Character c : line.toCharArray()) { if (i == pos) { newLine += Character.toString(escapeChar) + c; if (!rememberQuotePosition.isEmpty()) { pos = rememberQuotePosition.remove(0); } } else { newLine += c; } i++; } line = newLine; } return line; }
From source file:com.aiwoapp.crawler.writer.DBWriterProcessor.java
protected void validate(Pattern pat, String s) { if (!pat.matcher(s).matches()) { throw new IllegalArgumentException("invalid value: " + s + " does not match " + pat.pattern()); }/*from ww w .j a va2 s . co m*/ }
From source file:org.finra.dm.service.helper.Hive13DdlGenerator.java
/** * Gets a list of Hive partitions. For single level partitioning, no auto-discovery of sub-partitions (sub-directories) is needed - the business object data * will be represented by a single Hive partition instance. For multiple level partitioning, this method performs an auto-discovery of all sub-partitions * (sub-directories) and creates a Hive partition object instance for each partition. * * @param businessObjectDataKey the business object data key. * @param autoDiscoverableSubPartitionColumns the auto-discoverable sub-partition columns. * @param s3KeyPrefix the S3 key prefix. * @param storageFiles the storage files. * @param businessObjectDataEntity the business object data entity. * @param storageName the storage name.//from www. ja va2 s.c om * * @return the list of Hive partitions */ public List<HivePartitionDto> getHivePartitions(BusinessObjectDataKey businessObjectDataKey, List<SchemaColumn> autoDiscoverableSubPartitionColumns, String s3KeyPrefix, List<String> storageFiles, BusinessObjectDataEntity businessObjectDataEntity, String storageName) { // We are using linked hash map to preserve the order of the discovered partitions. LinkedHashMap<List<String>, HivePartitionDto> linkedHashMap = new LinkedHashMap<>(); Pattern pattern = getHivePathPattern(autoDiscoverableSubPartitionColumns); for (String storageFile : storageFiles) { // Remove S3 key prefix from the file path. Please note that the storage files are already validated to start with S3 key prefix. String relativeFilePath = storageFile.substring(s3KeyPrefix.length()); // Try to match the relative file path to the expected subpartition folders. Matcher matcher = pattern.matcher(relativeFilePath); Assert.isTrue(matcher.matches(), String.format( "Registered storage file or directory does not match the expected Hive sub-directory pattern. " + "Storage: {%s}, file/directory: {%s}, business object data: {%s}, S3 key prefix: {%s}, pattern: {^%s$}", storageName, storageFile, dmDaoHelper.businessObjectDataEntityAltKeyToString(businessObjectDataEntity), s3KeyPrefix, pattern.pattern())); // Add the top level partition value. HivePartitionDto newHivePartition = new HivePartitionDto(); newHivePartition.getPartitionValues().add(businessObjectDataKey.getPartitionValue()); newHivePartition.getPartitionValues().addAll(businessObjectDataKey.getSubPartitionValues()); // Extract relative partition values. for (int i = 1; i <= matcher.groupCount(); i++) { newHivePartition.getPartitionValues().add(matcher.group(i)); } // Remove the trailing "/" plus an optional file name from the file path and store the result string as this partition relative path. newHivePartition.setPath(relativeFilePath.replaceAll("/[^/]*$", "")); // Check if we already have that partition discovered - that would happen if partition contains multiple data files. HivePartitionDto hivePartition = linkedHashMap.get(newHivePartition.getPartitionValues()); if (hivePartition != null) { // Partition is already discovered, so just validate that the relative file paths match. Assert.isTrue(hivePartition.getPath().equals(newHivePartition.getPath()), String.format( "Found two different locations for the same Hive partition. Storage: {%s}, business object data: {%s}, " + "S3 key prefix: {%s}, path[1]: {%s}, path[2]: {%s}", storageName, dmDaoHelper.businessObjectDataEntityAltKeyToString(businessObjectDataEntity), s3KeyPrefix, hivePartition.getPath(), newHivePartition.getPath())); } else { // Add this partition to the hash map of discovered partitions. linkedHashMap.put(newHivePartition.getPartitionValues(), newHivePartition); } } List<HivePartitionDto> hivePartitions = new ArrayList<>(); hivePartitions.addAll(linkedHashMap.values()); return hivePartitions; }
From source file:adalid.commons.velocity.Writer.java
private void deletePreviouslyGeneratedFiles(String name, String[] stringArray, boolean recursive) { String root = getRoot().getPath(); String raiz = root.replace('\\', '/'); String path, pathname, wildcard; String recursively = recursive ? "and all its subdirectories" : ""; String slashedPath, regex, pattern, message; String hint = "; check property: {0}={1}"; boolean delete; File directory;//www .ja v a 2 s. c om IOFileFilter fileFilter; IOFileFilter dirFilter = recursive ? ignoreVersionControlFilter : null; Collection<File> matchingFiles; Arrays.sort(stringArray); for (String string : stringArray) { pathname = StringUtils.substringBeforeLast(string, SLASH); if (StringUtils.isBlank(string) || !StringUtils.contains(string, SLASH) || StringUtils.isBlank(pathname)) { pattern = "directory is missing or invalid" + hint; message = MessageFormat.format(pattern, name, string); log(_alertLevel, message); warnings++; continue; } wildcard = StringUtils.substringAfterLast(string, SLASH); if (StringUtils.isBlank(wildcard)) { pattern = "wildcard is missing or invalid" + hint; message = MessageFormat.format(pattern, name, string); log(_alertLevel, message); warnings++; continue; } directory = new File(pathname); if (FilUtils.isNotWritableDirectory(directory)) { pattern = "{2} is not a writable directory" + hint; message = MessageFormat.format(pattern, name, string, StringUtils.removeStartIgnoreCase(pathname, raiz)); log(_alertLevel, message); warnings++; continue; } log(_detailLevel, "deleting files " + wildcard + " from " + StringUtils.removeStartIgnoreCase(pathname, raiz) + " " + recursively); fileFilter = new WildcardFileFilter(wildcard); matchingFiles = FileUtils.listFiles(directory, fileFilter, dirFilter); for (File file : matchingFiles) { path = file.getPath(); slashedPath = path.replace('\\', '/'); delete = true; for (Pattern fxp : filePreservationPatterns) { regex = fxp.pattern(); if (slashedPath.matches(regex)) { delete = false; pattern = "file {0} will not be deleted; it matches preservation expression \"{1}\""; message = MessageFormat.format(pattern, StringUtils.removeStartIgnoreCase(slashedPath, raiz), regex); log(_alertLevel, message); warnings++; break; } } if (delete) { logger.trace("deleting " + StringUtils.removeStartIgnoreCase(path, root)); FileUtils.deleteQuietly(file); } } } }
From source file:org.etudes.component.app.melete.MeleteUtil.java
public ArrayList findEmbedItemPattern(String checkforimgs) { ArrayList returnData = new ArrayList(); // a and link uses href, applet uses archive, object uses data Pattern p1 = Pattern.compile( "<[iI][mM][gG]\\s|<[aA]\\s|<[eE][mM][bB][eE][dD]\\s|<[sS][cC][rR][iI][pP][tT]\\s|<[lL][iI][nN][kK]\\s|<[aA][pP][pP][lL][eE][tT]\\s|<[oO][bB][jJ][eE][cC][tT]\\s"); Pattern pi = Pattern.compile(">|\\s[sS][rR][cC]\\s*="); Pattern pa = Pattern.compile(">|\\s[hH][rR][eE][fF]\\s*="); Pattern pa1 = Pattern.compile(">|\\s[aA][rR][cC][hH][iI][vV][eE]\\s*="); Pattern pd = Pattern.compile(">|\\s[dD][aA][tT][aA]\\s*="); Pattern ps = Pattern.compile("\\S"); Pattern pe = Pattern.compile("\\s|>"); int startSrc = 0; int endSrc = 0; String foundPattern = null;//from w w w . j a va2 s . c o m while (checkforimgs != null) { foundPattern = null; // look for <img or <a Matcher m = p1.matcher(checkforimgs); if (!m.find()) // found anything? break; checkforimgs = checkforimgs.substring(m.start()); // look for src= or href= if (checkforimgs.startsWith("<i") || checkforimgs.startsWith("<I") || checkforimgs.startsWith("<e") || checkforimgs.startsWith("<E") || checkforimgs.startsWith("<s") || checkforimgs.startsWith("<S")) m = pi.matcher(checkforimgs); else if (checkforimgs.startsWith("<applet") || checkforimgs.startsWith("<Applet") || checkforimgs.startsWith("<APPLET")) m = pa1.matcher(checkforimgs); else if (checkforimgs.startsWith("<o") || checkforimgs.startsWith("<O")) m = pd.matcher(checkforimgs); else m = pa.matcher(checkforimgs); if (m.pattern().pattern().equals(pa.pattern())) { if (checkforimgs.startsWith("<a") || checkforimgs.startsWith("<A")) foundPattern = "link"; } // end = start+1 means that we found a > // i.e. the attribute we're looking for isn't there if (!m.find() || (m.end() == m.start() + 1)) { // prevent infinite loop by consuming the < checkforimgs = checkforimgs.substring(1); continue; } checkforimgs = checkforimgs.substring(m.end()); // look for start of arg, a non-whitespace m = ps.matcher(checkforimgs); if (!m.find()) // found anything? break; checkforimgs = checkforimgs.substring(m.start()); startSrc = 0; endSrc = 0; // handle either quoted or nonquoted arg if (checkforimgs.startsWith("\"") || checkforimgs.startsWith("\'")) { String quotestr = checkforimgs.substring(0, 1); startSrc = 1; endSrc = checkforimgs.indexOf(quotestr, startSrc); break; } else { startSrc = 0; // ends with whitespace or > m = pe.matcher(checkforimgs); if (!m.find()) // found anything? continue; endSrc = m.start(); } } //while end if (foundPattern != null && foundPattern.equals("link")) { String anchorStr = checkforimgs.substring(startSrc, endSrc); anchorStr = anchorStr.trim(); if (anchorStr != null && (anchorStr.startsWith("#") || anchorStr.startsWith("mailto:"))) { checkforimgs = checkforimgs.substring(endSrc); if (checkforimgs != null) { ArrayList r = findEmbedItemPattern(checkforimgs); checkforimgs = (String) r.get(0); if (r.size() > 1 && ((Integer) r.get(2)).intValue() > 0) { startSrc = ((Integer) r.get(1)).intValue(); endSrc = ((Integer) r.get(2)).intValue(); foundPattern = (String) r.get(3); } else { startSrc = 0; endSrc = 0; } } } } returnData.add(checkforimgs); if (endSrc != 0) { returnData.add(new Integer(startSrc)); returnData.add(new Integer(endSrc)); returnData.add(foundPattern); } return returnData; }
From source file:me.Wundero.Ray.utils.TextUtils.java
/** * Replace regular expression in a LiteralText object's content. * /*ww w. ja v a2s .c o m*/ * Note: this method does NOT check between object nodes. So if there is a * match between the original and one of it's children, it is not parsed. * * @param original * The original text. * @param matcher * The pattern to search for in the content. * @param replacer * The function to call when a match is found in order to supply * a textual replacement. * @param useClickHover * Whether or not to parse click and hover actions as well. */ public static LiteralText replaceRegex(LiteralText original, Pattern matcher, Function<LiteralText, Optional<LiteralText>> replacer, boolean useClickHover) { if (original == null) { return null; } List<Text> children = original.getChildren(); LiteralText.Builder builder = original.toBuilder(); TextFormat f = builder.getFormat(); Optional<ClickAction<?>> cl = builder.getClickAction(); Optional<HoverAction<?>> ho = builder.getHoverAction(); Optional<ShiftClickAction<?>> sc = builder.getShiftClickAction(); builder.removeAll(); String content = builder.getContent(); if (matcher.matcher(content).find()) { Matcher m = matcher.matcher(content); if (m.matches()) { builder = replacer.apply(builder.build()).orElse(LiteralText.of("")).toBuilder(); } else { String[] parts = content.split(matcher.pattern()); if (parts.length == 0) { String c = content; builder = null; while ((m = m.reset(c)).find()) { String g = m.group(); LiteralText t = replacer.apply( (LiteralText) TextUtils.apply(LiteralText.builder(g).format(f), cl, ho, sc).build()) .orElse(LiteralText.of("")); if (builder == null) { builder = t.toBuilder(); } else { builder.append(t); } c = m.replaceFirst(""); } } else { if (content.startsWith(parts[0])) { List<String> alt = Utils.alternate(parts, allmatches(content, matcher)); LiteralText.Builder ob = builder; builder = (LiteralText.Builder) LiteralText.builder(); List<String> pz = Utils.al(parts, true); for (String s : alt) { if (pz.contains(s)) { LiteralText t = replaceRegex(bf(s, ob).build(), matcher, replacer, useClickHover); builder.append(t); } else { Optional<LiteralText> t = replacer.apply((LiteralText) TextUtils .apply(LiteralText.builder(s).format(f), cl, ho, sc).build()); builder.append(t.orElse(LiteralText.of(""))); } } } else { List<String> alt = Utils.alternate(allmatches(content, matcher), parts); LiteralText.Builder ob = builder; builder = (LiteralText.Builder) LiteralText.builder(); List<String> pz = Utils.al(parts, true); for (String s : alt) { if (pz.contains(s)) { builder.append(replaceRegex(bf(s, ob).build(), matcher, replacer, useClickHover)); } else { Optional<LiteralText> t = replacer.apply((LiteralText) TextUtils .apply(LiteralText.builder(s).format(f), cl, ho, sc).build()); builder.append(t.orElse(LiteralText.of(""))); } } } } } } else { if (builder.getClickAction().isPresent()) { boolean r = builder.getClickAction().get() instanceof ClickAction.RunCommand; String click = replaceRegexAction(builder.getClickAction().get(), (s) -> { if (!matcher.matcher(s).find()) { return s; } String out = matcher.matcher(s).replaceAll("%s"); String st = s; Matcher m = matcher.matcher(s); List<Object> b = Utils.al(); while ((m = m.reset(st)).find()) { String g = m.group(); b.add(replacer.apply( (LiteralText) TextUtils.apply(LiteralText.builder(g).format(f), cl, ho, sc).build()) .orElse(LiteralText.builder("").build()).toPlain()); st = m.replaceFirst(""); } out = format(out, b); return out; }, ""); if (!click.isEmpty()) { if (r) { builder.onClick(TextActions.runCommand(click)); } else { builder.onClick(TextActions.suggestCommand(click)); } } } if (builder.getHoverAction().isPresent()) { Text hover = replaceRegexAction(builder.getHoverAction().get(), (s) -> { String a = TextSerializers.FORMATTING_CODE.serialize(s); if (!matcher.matcher(a).find()) { return s; } String out = matcher.matcher(a).replaceAll("%s"); String st = a; Matcher m = matcher.matcher(a); List<Object> b = Utils.al(); while ((m = m.reset(st)).find()) { String g = m.group(); b.add(TextSerializers.FORMATTING_CODE.serialize(replacer.apply( (LiteralText) TextUtils.apply(LiteralText.builder(g).format(f), cl, ho, sc).build()) .orElse(LiteralText.builder("").build()))); st = m.replaceFirst(""); } out = format(out, b); return TextSerializers.FORMATTING_CODE.deserialize(out); }, Text.of()); if (!hover.isEmpty()) { builder.onHover(TextActions.showText(hover)); } } } builder.append(children.stream().filter(t -> lit(t)) .map(child -> replaceRegex((LiteralText) child, matcher, replacer, useClickHover)) .collect(RayCollectors.rayList())); return builder.build(); }
From source file:adalid.commons.velocity.Writer.java
private void writeFile(WriterContext templateWriterContext, File templatePropertiesFile) { VelocityContext fileContext = templateWriterContext.getVelocityContextClone(); TLB.setProgrammers(templateWriterContext.programmers); TLB.setWrapperClasses(templateWriterContext.wrapperClasses); Properties properties = mergeProperties(fileContext, templatePropertiesFile); putStrings(fileContext, properties); // String rootPath = getRootPath(fileContext); String userPath = pathString(USER_DIR); String temptype = StringUtils.defaultIfBlank(properties.getProperty(TP_TYPE), TP_TYPE_VELOCITY); String template = StringUtils.trimToNull(properties.getProperty(TP_TEMPLATE)); String filePath = StringUtils.trimToNull(properties.getProperty(TP_PATH)); String filePack = StringUtils.trimToNull(properties.getProperty(TP_PACKAGE)); String fileName = StringUtils.trimToNull(properties.getProperty(TP_FILE)); String preserve = StringUtils.trimToNull(properties.getProperty(TP_PRESERVE)); String charset1 = StringUtils.trimToNull(properties.getProperty(TP_ENCODING)); String charset2 = StringUtils.trimToNull(properties.getProperty(TP_CHARSET)); String root = getRoot().getPath(); String raiz = root.replace('\\', '/'); String hint = "; check property \"{0}\" at file \"{1}\""; if (ArrayUtils.contains(TP_TYPE_ARRAY, temptype)) { } else {//from w ww . j av a 2s . co m String pattern = "failed to obtain a valid template type" + hint; String message = MessageFormat.format(pattern, TP_TYPE, templatePropertiesFile); logger.error(message); errors++; return; } if (template == null) { String pattern = "failed to obtain a valid template name" + hint; String message = MessageFormat.format(pattern, TP_TEMPLATE, templatePropertiesFile); logger.error(message); errors++; return; } if (fileName == null) { String pattern = "failed to obtain a valid file name" + hint; String message = MessageFormat.format(pattern, TP_FILE, templatePropertiesFile); logger.error(message); errors++; return; } String templatePathString = pathString(template); String templatePath = StringUtils.substringBeforeLast(templatePathString, FILE_SEPARATOR); fileContext.put(VC_TEMPLATE, StringEscapeUtils.escapeJava(templatePathString)); fileContext.put(VC_TEMPLATE_PATH, StringUtils.replace(templatePath, FILE_SEPARATOR, SLASH)); fileContext.put(VC_FILE, fileName); if (filePath == null) { // filePath = rootPath; filePath = userPath; } else { filePath = pathString(filePath); if (isRelativePath(filePath)) { if (filePath.startsWith(FILE_SEPARATOR)) { // filePath = rootPath + filePath; filePath = userPath + filePath; } else { // filePath = rootPath + FILE_SEPARATOR + filePath; filePath = userPath + FILE_SEPARATOR + filePath; } } } fileContext.put(VC_PATH, StringEscapeUtils.escapeJava(filePath)); if (filePack != null) { filePath += FILE_SEPARATOR + pathString(StringUtils.replace(filePack, DOT, SLASH)); fileContext.put(VC_PACKAGE, dottedString(filePack)); } File path = new File(filePath); if (path.exists()) { if (path.isDirectory() && path.canWrite()) { } else { String pattern = "{2} is not a valid directory" + hint; String message = MessageFormat.format(pattern, TP_PATH, templatePropertiesFile, path); logger.error(message); errors++; return; } } else if (path.mkdirs()) { } else { String pattern = "{2} is not a valid path" + hint; String message = MessageFormat.format(pattern, TP_PATH, templatePropertiesFile, path); logger.error(message); errors++; return; } String fullname = path.getPath() + FILE_SEPARATOR + fileName; String slashedPath = fullname.replace('\\', '/'); File file = new File(fullname); if (file.exists()) { String regex, pattern, message; for (Pattern fxp : fileExclusionPatterns) { regex = fxp.pattern(); if (slashedPath.matches(regex)) { excludedFiles++; pattern = "file {0} will be deleted; it matches exclusion expression \"{1}\""; message = MessageFormat.format(pattern, StringUtils.removeStartIgnoreCase(slashedPath, raiz), regex); log(_alertLevel, message); warnings++; FileUtils.deleteQuietly(file); return; } } if (BitUtils.valueOf(preserve)) { preservedFiles++; pattern = "file {2} will not be replaced" + hint; message = MessageFormat.format(pattern, TP_PRESERVE, templatePropertiesFile, fullname); log(_detailLevel, message); return; } for (Pattern fxp : filePreservationPatterns) { regex = fxp.pattern(); if (slashedPath.matches(regex)) { preservedFiles++; pattern = "file {0} will not be replaced; it matches preservation expression \"{1}\""; message = MessageFormat.format(pattern, StringUtils.removeStartIgnoreCase(slashedPath, raiz), regex); log(_alertLevel, message); warnings++; return; } } } else { String regex, pattern, message; for (Pattern fxp : fileExclusionPatterns) { regex = fxp.pattern(); if (slashedPath.matches(regex)) { excludedFiles++; pattern = "file {0} will not be written; it matches exclusion expression \"{1}\""; message = MessageFormat.format(pattern, StringUtils.removeStartIgnoreCase(slashedPath, raiz), regex); log(_alertLevel, message); warnings++; return; } } } if (charset1 != null && !Charset.isSupported(charset1)) { String pattern = "{2} is not a supported character set" + hint; String message = MessageFormat.format(pattern, TP_ENCODING, templatePropertiesFile, charset1); logger.error(message); errors++; return; } if (charset2 != null && !Charset.isSupported(charset2)) { String pattern = "{2} is not a supported character set" + hint; String message = MessageFormat.format(pattern, TP_CHARSET, templatePropertiesFile, charset2); logger.error(message); errors++; return; } fileContext.put(VC_FILE_PATH, StringEscapeUtils.escapeJava(filePath)); fileContext.put(VC_FILE_NAME, StringEscapeUtils.escapeJava(fileName)); fileContext.put(VC_FILE_PATH_NAME, StringEscapeUtils.escapeJava(fullname)); fileContext.put(VC_FILE_PATH_FILE, path); if (temptype.equals(TP_TYPE_VELOCITY)) { writeFile(fileContext, template, fullname, charset1, charset2); } else { writeFile(template, fullname); } executeFile(fileContext, templatePropertiesFile); }
From source file:edu.brown.workload.Workload.java
/** * // www . j av a 2 s .c o m * @param input_path * @param catalog_db * @param limit * @throws Exception */ public Workload load(File input_path, Database catalog_db, Filter filter) throws Exception { if (debug.val) LOG.debug("Reading workload trace from file '" + input_path + "'"); this.input_path = input_path; long start = System.currentTimeMillis(); // HACK: Throw out traces unless they have the procedures that we're looking for Pattern temp_pattern = null; if (filter != null) { List<ProcedureNameFilter> procname_filters = filter.getFilters(ProcedureNameFilter.class); if (procname_filters.isEmpty() == false) { Set<String> names = new HashSet<String>(); for (ProcedureNameFilter f : procname_filters) { for (String name : f.getProcedureNames()) { names.add(Pattern.quote(name)); } // FOR } // FOR if (names.isEmpty() == false) { temp_pattern = Pattern.compile( String.format("\"NAME\":[\\s]*\"(%s)\"", StringUtil.join("|", names)), Pattern.CASE_INSENSITIVE); if (debug.val) { LOG.debug(String.format("Fast filter for %d procedure names", names.size())); LOG.debug("PATTERN: " + temp_pattern.pattern()); } } } } final Pattern pattern = temp_pattern; final AtomicInteger counters[] = new AtomicInteger[] { new AtomicInteger(0), // ELEMENT COUNTER new AtomicInteger(0), // TXN COUNTER new AtomicInteger(0), // QUERY COUNTER new AtomicInteger(0), // WEIGHTED TXN COUNTER new AtomicInteger(0), // WEIGHTED QUERY COUNTER }; List<Runnable> all_runnables = new ArrayList<Runnable>(); int num_threads = ThreadUtil.getMaxGlobalThreads(); // Create the reader thread first WorkloadUtil.ReadThread rt = new WorkloadUtil.ReadThread(this.input_path, pattern, num_threads); all_runnables.add(rt); // Then create all of our processing threads for (int i = 0; i < num_threads; i++) { WorkloadUtil.ProcessingThread lt = new WorkloadUtil.ProcessingThread(this, this.input_path, rt, catalog_db, filter, counters); rt.processingThreads.add(lt); all_runnables.add(lt); } // FOR if (debug.val) LOG.debug(String.format("Loading workload trace using %d ProcessThreads", rt.processingThreads.size())); ThreadUtil.runNewPool(all_runnables, all_runnables.size()); VerifyWorkload.verify(catalog_db, this); long stop = System.currentTimeMillis(); LOG.info(String.format("Loaded %d txns / %d queries from '%s' in %.1f seconds using %d threads", this.txn_traces.size(), counters[2].get(), this.input_path.getName(), (stop - start) / 1000d, num_threads)); if (counters[1].get() != counters[3].get() || counters[2].get() != counters[4].get()) { LOG.info( String.format("Weighted Workload: %d txns / %d queries", counters[3].get(), counters[4].get())); } return (this); }
From source file:com.gitblit.utils.JGitUtils.java
/** * Recursive function to find git repositories. * * @param basePath/*from w w w . j a v a 2 s .c o m*/ * basePath is stripped from the repository name as repositories * are relative to this path * @param searchFolder * @param onlyBare * if true only bare repositories will be listed. if false all * repositories are included. * @param searchSubfolders * recurse into subfolders to find grouped repositories * @param depth * recursion depth, -1 = infinite recursion * @param patterns * list of regex patterns for matching to folder names * @return */ private static List<String> getRepositoryList(String basePath, File searchFolder, boolean onlyBare, boolean searchSubfolders, int depth, List<Pattern> patterns) { File baseFile = new File(basePath); List<String> list = new ArrayList<String>(); if (depth == 0) { return list; } int nextDepth = (depth == -1) ? -1 : depth - 1; for (File file : searchFolder.listFiles()) { if (file.isDirectory()) { boolean exclude = false; for (Pattern pattern : patterns) { String path = FileUtils.getRelativePath(baseFile, file).replace('\\', '/'); if (pattern.matcher(path).matches()) { LOGGER.debug( MessageFormat.format("excluding {0} because of rule {1}", path, pattern.pattern())); exclude = true; break; } } if (exclude) { // skip to next file continue; } File gitDir = FileKey.resolve(new File(searchFolder, file.getName()), FS.DETECTED); if (gitDir != null) { if (onlyBare && gitDir.getName().equals(".git")) { continue; } if (gitDir.equals(file) || gitDir.getParentFile().equals(file)) { // determine repository name relative to base path String repository = FileUtils.getRelativePath(baseFile, file); list.add(repository); } else if (searchSubfolders && file.canRead()) { // look for repositories in subfolders list.addAll( getRepositoryList(basePath, file, onlyBare, searchSubfolders, nextDepth, patterns)); } } else if (searchSubfolders && file.canRead()) { // look for repositories in subfolders list.addAll(getRepositoryList(basePath, file, onlyBare, searchSubfolders, nextDepth, patterns)); } } } return list; }
From source file:com.clustercontrol.hub.util.CollectStringDataParser.java
/** * ????????????/* w w w . j a va2 s . c o m*/ * * @param data * @return */ public CollectStringData parse(CollectStringData data) { Map<String, CollectDataTag> tagMap = new HashMap<>(); for (CollectDataTag tag : data.getTagList()) { tagMap.put(tag.getKey(), tag); } if (isNullOrZeroLength(format.getTimestampRegex()) && isNullOrZeroLength(format.getTimestampFormat())) { // do nothing, use currentTimeMillis as timestamp } else { Matcher m = timestampPattern.matcher(data.getValue()); if (m.find() && m.groupCount() > 0) { String timestampStr = m.group(1); try { DateTime datetime = timestampFormatter.parseDateTime(timestampStr); if (datetime.year().get() == 0) { // for messages without year, like syslog DateTime now = new DateTime(); DateTimeFormatter timestampFormatterWithCurrentYear = timestampFormatter .withDefaultYear(now.year().get()); DateTimeFormatter timestampFormatterWithLastYear = timestampFormatter .withDefaultYear(now.year().get() - 1); datetime = timestampFormatterWithCurrentYear.parseDateTime(timestampStr); if (datetime.getMillis() - now.getMillis() > 1000 * 60 * 60 * 24 * 7) { // treat messages as end of year (threshold : 1 week) datetime = timestampFormatterWithLastYear.parseDateTime(timestampStr); } } tagMap.put(KEY_TIMESTAMP_IN_LOG, new CollectDataTag( new CollectDataTagPK(data.getCollectId(), data.getDataId(), KEY_TIMESTAMP_IN_LOG), ValueType.number, Long.toString(datetime.getMillis()))); } catch (IllegalArgumentException e) { log.warn(String.format("invalid timestamp string : format = %s, string = %s", format.getTimestampRegex(), timestampStr)); } } } for (LogFormatKey keyword : format.getKeyPatternList()) { Pattern p = keywordPatternMap.get(keyword.getKey()); if (null == p) { log.debug(String.format("Pattern is null keyword : pattern=%s", keyword.getPattern())); continue; } Matcher m = p.matcher(data.getValue()); String matchedStr = null; switch (keyword.getKeyType()) { case parsing: if (m.find() && m.groupCount() > 0) { matchedStr = m.group(1); } break; case fixed: if (m.find()) { matchedStr = keyword.getValue(); } break; } if (matchedStr != null && keyword.getValueType() == ValueType.string) { tagMap.put(keyword.getKey(), new CollectDataTag( new CollectDataTagPK(data.getCollectId(), data.getDataId(), keyword.getKey()), keyword.getValueType(), matchedStr)); } else if (matchedStr != null && keyword.getValueType() != ValueType.string) { tagMap.put(keyword.getKey(), new CollectDataTag( new CollectDataTagPK(data.getCollectId(), data.getDataId(), keyword.getKey()), keyword.getValueType(), matchedStr)); switch (keyword.getValueType()) { case number: try { new BigDecimal(matchedStr); } catch (NumberFormatException e) { log.warn(String.format("not match number format : value=%s, source=%s, pattern=%s", matchedStr, data.getValue(), p.pattern())); } break; case bool: if (!"true".equalsIgnoreCase(matchedStr) || !"false".equalsIgnoreCase(matchedStr)) { log.warn(String.format("not match boolean type : value=%s, source=%s, pattern=%s", matchedStr, data.getValue(), p.pattern())); } break; default: log.warn(String.format("unexpected value type : type=%s, value=source=%s, pattern=%s", keyword.getValueType().name(), data.getValue(), p.pattern())); break; } } } data.setTagList(new ArrayList<>(tagMap.values())); return data; }