List of usage examples for org.apache.commons.io FileUtils iterateFiles
public static Iterator iterateFiles(File directory, String[] extensions, boolean recursive)
From source file:de.mprengemann.intellij.plugin.androidicons.dialogs.AndroidMultiDrawableImporter.java
private void importZipArchive(VirtualFile virtualFile) { final String filePath = virtualFile.getCanonicalPath(); if (filePath == null) { return;/*from w w w. j av a 2 s .c o m*/ } final File tempDir = new File(ImageInformation.getTempDir(), virtualFile.getNameWithoutExtension()); final String archiveName = virtualFile.getName(); new Task.Modal(project, "Importing Archive...", true) { @Override public void run(@NotNull final ProgressIndicator progressIndicator) { progressIndicator.setIndeterminate(true); try { FileUtils.forceMkdir(tempDir); ZipUtil.extract(new File(filePath), tempDir, new FilenameFilter() { @Override public boolean accept(File dir, String name) { final String mimeType = new MimetypesFileTypeMap().getContentType(name); final String type = mimeType.split("/")[0]; return type.equals("image"); } }, true); progressIndicator.checkCanceled(); final Iterator<File> fileIterator = FileUtils.iterateFiles(tempDir, TrueFileFilter.INSTANCE, TrueFileFilter.INSTANCE); while (fileIterator.hasNext()) { File file = fileIterator.next(); if (file.isDirectory() || file.isHidden()) { continue; } final String fileRoot = file.getParent().toUpperCase(); final String name = FilenameUtils.getBaseName(file.toString()); if (name.startsWith(".") || fileRoot.contains("__MACOSX")) { continue; } for (Resolution resolution : RESOLUTIONS) { if (name.toUpperCase().contains("-" + resolution) || name.toUpperCase().contains("_" + resolution) || fileRoot.contains(resolution.toString())) { controller.addZipImage(file, resolution); break; } } } progressIndicator.checkCanceled(); final Map<Resolution, List<ImageInformation>> zipImages = controller.getZipImages(); final List<Resolution> foundResolutions = new ArrayList<Resolution>(); int foundAssets = 0; for (Resolution resolution : zipImages.keySet()) { final List<ImageInformation> assetInformation = zipImages.get(resolution); if (assetInformation != null && assetInformation.size() > 0) { foundAssets += assetInformation.size(); foundResolutions.add(resolution); } } progressIndicator.checkCanceled(); final int finalFoundAssets = foundAssets; UIUtil.invokeLaterIfNeeded(new DumbAwareRunnable() { public void run() { final String title = String.format("Import '%s'", archiveName); if (foundResolutions.size() == 0 || finalFoundAssets == 0) { Messages.showErrorDialog("No assets found.", title); FileUtils.deleteQuietly(tempDir); return; } final String[] options = new String[] { "Import", "Cancel" }; final String description = String.format("Import %d assets for %s to %s.", finalFoundAssets, StringUtils.join(foundResolutions, ", "), controller.getTargetRoot()); final int selection = Messages.showDialog(description, title, options, 0, Messages.getQuestionIcon()); if (selection == 0) { controller.getZipTask(project, tempDir).queue(); close(0); } else { FileUtils.deleteQuietly(tempDir); } } }); } catch (ProcessCanceledException e) { FileUtils.deleteQuietly(tempDir); } catch (IOException e) { LOGGER.error(e); } } }.queue(); }
From source file:at.ofai.gate.virtualcorpus.DirectoryCorpus.java
/** * Initializes the DirectoryCorpus LR/*from w w w. ja v a 2s . com*/ * @return * @throws ResourceInstantiationException */ @Override public Resource init() throws ResourceInstantiationException { logger.info("DirectoryCorpus: calling init"); if (directoryURL == null) { throw new ResourceInstantiationException("directoryURL must be set"); } // first of all, create a map that contains all the supported extensions // as keys and the corresponding documente exporter as value. // First, get all the supported extensions for reading files Set<String> readExtensions = DocumentFormat.getSupportedFileSuffixes(); logger.info("DirectoryCorpus/init readExtensions=" + readExtensions); Set<String> supportedExtensions = new HashSet<String>(); // if we also want to write, we have to limit the supported extensions // to those where we have an exporter and also we need to remember which // exporter supports which extensions if (!getReadonly()) { List<Resource> des = null; try { // Now get all the Document exporters des = Gate.getCreoleRegister().getAllInstances("gate.DocumentExporter"); } catch (GateException ex) { throw new ResourceInstantiationException("Could not get the document exporters", ex); } for (Resource r : des) { DocumentExporter d = (DocumentExporter) r; if (readExtensions.contains(d.getDefaultExtension())) { extension2Exporter.put(d.getDefaultExtension(), d); supportedExtensions.add(d.getDefaultExtension()); } } } else { supportedExtensions.addAll(readExtensions); } logger.info("DirectoryCorpus/init supportedExtensions=" + readExtensions); // now check if an extension list was specified by the user. If no, nothing // needs to be done. If yes, remove all the extensions from the extnesion2Exporter // map which were not specified and warn about all the extensions specified // for which we do not have an entry. Also remove them from the supportedExtensions set if (getExtensions() != null && !getExtensions().isEmpty()) { logger.info("DirectoryCorpu/init getExtgension is not empty: " + getExtensions()); for (String ext : getExtensions()) { if (!supportedExtensions.contains(ext)) { logger.warn("DirectoryCorpus warning: extension is not supported: " + ext); } } // now remove all the extensions which are not specified Iterator<String> it = supportedExtensions.iterator(); while (it.hasNext()) { String ext = it.next(); logger.info("DirectoryCorpus/init checking supported extension: " + ext); if (!getExtensions().contains(ext)) { logger.info("DirectoryCorpus/init removing extension: " + ext); it.remove(); extension2Exporter.remove(ext); } } } logger.info("DirectoryCorpus/init supportedExtensions after parms: " + supportedExtensions); logger.info("DirectoryCorpus/init exporter map: " + extension2Exporter); if (supportedExtensions.isEmpty()) { throw new ResourceInstantiationException( "DirectoryCorpus could not be created, no file format supported or loaded"); } backingDirectoryFile = Files.fileFromURL(directoryURL); try { backingDirectoryFile = backingDirectoryFile.getCanonicalFile(); } catch (IOException ex) { throw new ResourceInstantiationException("Cannot get canonical file for " + backingDirectoryFile, ex); } if (!backingDirectoryFile.isDirectory()) { throw new ResourceInstantiationException("Not a directory " + backingDirectoryFile); } try { ourDS = (DummyDataStore4DirCorp) Factory.createDataStore( "at.ofai.gate.virtualcorpus.DummyDataStore4DirCorp", backingDirectoryFile.getAbsoluteFile().toURI().toURL().toString()); ourDS.setName("DummyDS4_" + this.getName()); ourDS.setComment("Dummy DataStore for DirectoryCorpus " + this.getName()); ourDS.setCorpus(this); //System.err.println("Created dummy corpus: "+ourDS+" with name "+ourDS.getName()); } catch (Exception ex) { throw new ResourceInstantiationException("Could not create dummy data store", ex); } logger.info("DirectoryCorpus/init: ds created: " + ourDS.getName()); Iterator<File> fileIt = FileUtils.iterateFiles(backingDirectoryFile, supportedExtensions.toArray(new String[0]), getRecurseDirectory()); int i = 0; while (fileIt.hasNext()) { File file = fileIt.next(); // if recursion was specified, we need to get the relative file path // relative to the root directory. This is done by getting the canonical // full path name for both the directory and the file and then // relativizing the path. String filename = file.getName(); // TODO: first check if this file should be ignored (hidden files?) if (!filename.startsWith(".")) { if (getRecurseDirectory()) { try { file = file.getCanonicalFile(); } catch (IOException ex) { throw new ResourceInstantiationException("Could not get canonical path for " + file); } filename = backingDirectoryFile.toURI().relativize(file.toURI()).getPath(); } documentNames.add(filename); isLoadeds.add(false); documentIndexes.put(filename, i); i++; } } if (i == 0) { logger.warn("DirectoryCorpus warning: empty immutable corpus created, no files found"); } try { PersistenceManager.registerPersistentEquivalent(at.ofai.gate.virtualcorpus.DirectoryCorpus.class, at.ofai.gate.virtualcorpus.DirectoryCorpusPersistence.class); } catch (PersistenceException e) { throw new ResourceInstantiationException("Could not register persistence", e); } Gate.getCreoleRegister().addCreoleListener(this); return this; }
From source file:es.bsc.servicess.ide.PackagingUtils.java
/** Copy the runtime files required for the Core Elements. * @param runtimePath Path to the programming model runtime * @param jarFolder Folder to store the jars *//*from ww w . j a va 2s.c o m*/ private static void copyCoreRuntimeFiles(String runtimePath, IFolder jarFolder) { File d = new File(runtimePath + "/../worker/"); File it_jar = new File(runtimePath + File.separator + "lib" + File.separator + "IT.jar"); File jar_folder_file = jarFolder.getLocation().toFile(); File jar_lib_folder = jarFolder.getLocation().append("lib").toFile(); try { FileUtils.copyFileToDirectory(it_jar, jar_lib_folder); } catch (IOException e) { log.error("File " + it_jar.getAbsolutePath() + "could not be copied to " + jar_lib_folder.getAbsolutePath()); } if (d.isDirectory()) { Iterator<File> fi = FileUtils.iterateFiles(d, new String[] { "sh" }, false); while (fi.hasNext()) { File f = fi.next(); if (!isFileInDiscardList(f)) { try { // System.out.println("Trying to copy File "+ // f.getAbsolutePath()); FileUtils.copyFileToDirectory(f, jar_folder_file); File fc = new File(jar_folder_file.getAbsolutePath() + File.separator + f.getName()); fc.setExecutable(true); log.debug(" File copied " + f.getAbsolutePath()); } catch (IOException e) { log.error("File " + f.getAbsolutePath() + "could not be copied to " + jar_folder_file.getAbsolutePath()); } } } } else log.error("File " + d.getAbsolutePath() + "is not a directory"); }
From source file:controllers.ArtifactsController.java
/** * Protected so that it does not fire a render event... *///from w w w. j a v a2 s. c om protected static Result deleteLocalArtifactFiles() { Iterator<File> iter = FileUtils.iterateFiles(getArtifactsFolder(), new String[] { "zip" }, false); while (iter.hasNext()) { System.out.println("TODO : Delete " + iter.next()); //FileUtils.deleteQuietly(iter.next()); } return repository(); }
From source file:de.icongmbh.oss.maven.plugin.javassist.JavassistTransformerExecutor.java
/** * Search for class files (file extension: {@code .class}) on the passed {@code directory}. * <p>//from www .j av a2 s . c om * Note: The passed directory name must exist and readable. * </p> * * @param directory must nor be {@code null} * @return iterator of full qualified class names and never {@code null} * * @throws NullPointerException if passed {@code directory} is {@code null}. * * @see SuffixFileFilter * @see TrueFileFilter * @see FileUtils#iterateFiles(File, IOFileFilter, IOFileFilter) * @see ClassnameExtractor#iterateClassnames(File, Iterator) */ protected Iterator<String> iterateClassnames(final String directory) { final File dir = new File(directory); final String[] extensions = { ".class" }; final IOFileFilter fileFilter = new SuffixFileFilter(extensions); final IOFileFilter dirFilter = TrueFileFilter.INSTANCE; return ClassnameExtractor.iterateClassnames(dir, FileUtils.iterateFiles(dir, fileFilter, dirFilter)); }
From source file:de.tarent.maven.plugins.pkg.Utils.java
/** * Copies the <code>AuxFile</code> instances contained within the set. It * takes the <code>srcAuxFilesDir</code> and <code>auxFileDstDir</code> * arguments into account to specify the parent source and destination * directory of the files.//from w w w. j a v a 2 s .co m * * By default files are copied into directories. If the <code>rename</code> * property of the <code>AuxFile</code> instance is set however the file is * copied and renamed to the last part of the path. * * The return value is the amount of copied bytes. * * @param l * @param srcAuxFilesDir * @param dstDir * @param auxFiles * @param makeExecutable * @return * @throws MojoExecutionException */ public static long copyFiles(Log l, File srcDir, File dstDir, List<? extends AuxFile> auxFiles, String type, boolean makeExecutable) throws MojoExecutionException { long size = 0; Iterator<? extends AuxFile> ite = auxFiles.iterator(); while (ite.hasNext()) { AuxFile af = (AuxFile) ite.next(); File from = new File(srcDir, af.from); File to = new File(dstDir, af.to); l.info("copying " + type + ": " + from.toString()); l.info("destination: " + to.toString()); if (!from.exists()) { throw new MojoExecutionException("File to copy does not exist: " + from.toString()); } createParentDirs(to, type); try { if (from.isDirectory()) { to = new File(to, from.getName()); FileUtils.copyDirectory(from, to, FILTER); for (final Iterator<File> files = FileUtils.iterateFiles(from, FILTER, FILTER); files .hasNext();) { final File nextFile = files.next(); size += nextFile.length(); } } else if (af.isRename()) { FileUtils.copyFile(from, to); size += from.length(); if (makeExecutable) { makeExecutable(l, to.getAbsolutePath()); } } else { FileUtils.copyFileToDirectory(from, to); size += from.length(); if (makeExecutable) { makeExecutable(l, to.getAbsolutePath() + File.separator + from.getName()); } } } catch (IOException ioe) { throw new MojoExecutionException("IOException while copying " + type, ioe); } } return size; }
From source file:edu.cmu.cs.lti.discoursedb.io.mturk.converter.MturkConverter.java
License:asdf
private void convert(String directory, String datasetName) throws ParseException, IOException { // xu_end_id is string: group + _ team + _ + id // username is group:username Map<String, String> xu_id2username = new HashMap<String, String>(); Map<String, String> wen_username2groupteam = new HashMap<String, String>(); Map<String, String> wen_wenHandle2discId = new HashMap<String, String>(); Map<String, String> discId2expHandle = new HashMap<String, String>(); Map<String, Long> ddb_user_ids = new HashMap<String, Long>(); Map<String, String> discId2discHandle = new HashMap<String, String>(); Map<String, String> discHandle2discId = new HashMap<String, String>(); Map<String, String> expHandle2group = new HashMap<String, String>(); Map<String, String> expHandle2team = new HashMap<String, String>(); Map<String, String> expHandle2experiment = new HashMap<String, String>(); Boolean summerSchool = true;/*ww w . j av a 2s. c o m*/ Pattern forum_team_user0 = Pattern.compile("(\\d\\d\\d)_(\\d+)_(\\d+)"); Matcher m11 = forum_team_user0.matcher("234pre234_2_3.csv"); m11.find(); assert m11.group(1) == "234"; System.out.println("Success!"); /* * Read xu and wen's users -> write users, * keep map xu_user_id -> username, group, team, experiment; username->xu_user_id,group, team, experiment; also username -> discoursedb_userid * write group entities for all three levels and link to users * write DPs for group, team, experiment, and link them (dprs) */ /* userid,groupid,group,newuserid,username,id,trans,bazaar,teamid,rolereasoning, * chattrans,score,tradeoff,discussiontran,totalprelen,reasoning,chatlength, * numofwords,reasoning_percentage,bazaarprompt,tranpercentage,pre_epi_rt, * pre_epi_wr,pre_tf_rt,pre_tf_wr,post_epi_rt,post_epi_wr,post_tf_rt,post_tf_wr * * 1,222_1,222,222_1,Bobs,222_1_1,1,1,mturk987641,4,10,23... */ //for (Map<String,String> row : csvIteratorExistingHeaders(directory + "/xustudy/individualdata_0622.csv")) { for (Map<String, String> row : csvIteratorExistingHeaders( directory + "/summerschool/individualuser_0710.csv")) { String group = row.get("id").split("_")[0]; String team = row.get("id").split("_")[1]; String groupteam = group + "_" + team; String xuHandle = group + ":" + row.get("username"); xu_id2username.put(row.get("id"), xuHandle); ddb_user_ids.put(xuHandle, mcs.mapUser(xuHandle, xuDiscourseName, datasetName, "individualdata_0622", "id", row.get("id"))); expHandle2group.put(xuHandle, group); expHandle2team.put(xuHandle, team); expHandle2experiment.put(xuHandle, xuDiscourseName); mcs.mapTeamAndGroup(xuDiscourseName, group, team, datasetName, "individualdata_0622", "id", row.get("id")); } /* discforum2wenstudy.csv * * wenHandle,discId,forum,discName c1:Shan,726,10,shan c1:StickyWicket,707,10,StickyWicket c1:WT89,701,10,WT89 c1:hjo,712,10,hjo */ if (!summerSchool) { for (Map<String, String> row : csvIteratorExistingHeaders(directory + "/discforum2wenstudy.csv")) { String groupteam = row.get("wenHandle").split(":")[0]; String group = wenGroup(groupteam); String team = wenTeam(groupteam); if (row.get("discId") == "") { System.out .println("Skipping user " + row.get("wenHandle") + ": discussionforum id is not known"); continue; } String wenHandle = row.get("wenHandle"); discId2expHandle.put(row.get("discId"), wenHandle); wen_wenHandle2discId.put(wenHandle, row.get("discId")); ddb_user_ids.put(wenHandle, mcs.mapUser(wenHandle, wenDiscourseName, datasetName, "discforum2wenstudy", "discId", row.get("discId"))); expHandle2group.put(wenHandle, group); expHandle2team.put(wenHandle, team); expHandle2experiment.put(wenHandle, wenDiscourseName); mcs.mapTeamAndGroup(wenDiscourseName, group, team, datasetName, "discforum2wenstudy", "id", row.get("discId")); } } /* * userid,assign_groupsize,groupid,totalpost,cothreadwithteammates,gotreplyfromteammate,replytoteammate,initialproposal,team_score,team_energy_requirement,team_energy_energy,team_additional,team_incorrect,score,energy_requirement,energy_energy,additional,incorrect,cntwords,cntchats,Experience,End_Result,Communication_Quality,Topic_Familiarity,Perceived_Learning,type,energy * * Amy,3,ff1,6,0,0,0,226,2,2,0,0,0,0,0,0,0,0,0,0,5,5,5,2,5,community-early,1 * for (Map<String,String> row : csvIteratorExistingHeaders(directory + "/wenstudy/exp1-ANOVA-peruser.csv")) { String group = row.get("groupid").substring(0, row.get("groupid").length()-1); String username = group + ":" + row.get("userid"); wen_username2groupteam.put(username, row.get("groupid")); username2group.put(username, group); String team = row.get("groupid"); username2team.put(username, team); username2experiment.put(username, wenDiscourseName); ddb_user_ids.put(username, mcs.mapUser(username, wenDiscourseName, datasetName, "exp1-ANOVA-peruser", "userid", row.get("userid"))); //, team, group, "xustudy")); System.out.println(row.get("userid") + " --> " + row.get("groupid")); mcs.mapTeamAndGroup(wenDiscourseName, group, team, datasetName, "individualdata_0622", "groupid", row.get("groupid")); }*/ /* Table<R, C, V> t2 = csvscan("wenfiles/exp1_sdmething.csv", new Array<String>(flds)); for (t in t2.rows()) { ddb_user_ids[t.get("user_name")] = mcs.addUser(t.get("userxxxxxid"), t.get("user_name"), group, team, experiment); // also fields here } //* Read users.csv -> keep discussion_id in memory; don't write * "user_uid","user_name","user_pwd","forum_uid","uuid","access_enabled","password_reset" * * "1","erose","innerpath","1","CF5725C5-B089-CC1F-509F4E3E9BE24881","1","" * "2603","Amber","Amber64","64","Amber","1","" * "173","64","64","1","64","1","" */ //for (Map<String,String> row : csvIteratorExistingHeaders(directory + "/user.csv")) { for (Map<String, String> row : csvIteratorExistingHeaders(directory + "summerschool/user0710.csv")) { discId2discHandle.put(row.get("user_uid"), row.get("forum_uid") + ":" + row.get("user_name")); discHandle2discId.put(row.get("forum_uid") + ":" + row.get("user_name"), row.get("user_uid")); } /* discussionforum.csv * post_uid,forum_uid,thread_uid,replyto_uid,user_uid,subject,content, * posted_at,uuid * * 15,1,7,0,3,"If Oil is Scarce, Why's It So Damn Cheap?","My question ...", * 4/4/15 20:20,4584DA50-EDFC-B74D-EEAAA78C8CF4F2DC */ Map<Long, Long> sourceDiscId2ddbDiscId = new HashMap<Long, Long>(); for (Map<String, String> row : csvIteratorExistingHeaders(directory + "/summerschool/forum0710.csv")) { String discHandle = discId2discHandle.getOrDefault(row.get("user_uid"), row.get("forum_uid") + ":User" + row.get("user_uid")); String expHandle = discId2expHandle.getOrDefault(row.get("user_uid"), discHandle); String thisDiscourse = expHandle2experiment.getOrDefault(expHandle, "discussionforum"); if (!ddb_user_ids.containsKey(expHandle)) { ddb_user_ids.put(expHandle, mcs.mapUser(expHandle, thisDiscourse, datasetName, "discussionforum", "post_uid(User)", row.get("post_uid"))); } Long post_uid = Long.valueOf(row.get("post_uid")); System.out.println("Mapping post " + row.get("post_uid") + " by user " + expHandle + " aka " + sourceDiscId2ddbDiscId.getOrDefault(Long.valueOf(row.get("user_uid")), 0L)); Long post_ddbid = mcs.mapDiscussionPost(row.get("subject"), row.get("content"), row.get("forum_uid"), row.get("thread_uid"), expHandle2group.get(expHandle), expHandle2team.get(expHandle), ddb_user_ids.getOrDefault(expHandle, 0L), row.get("posted_at"), Long.valueOf(row.get("replyto_uid")), thisDiscourse, datasetName, "discussionforum", "post_uid", row.get("post_uid")); sourceDiscId2ddbDiscId.put(post_uid, post_ddbid); } /* * forumid, offset, forumname * 218,10510,92722 * 222,10810,98764 * 224,11010,79865 */ Map<String, String> xu_forumname2forum = new HashMap<String, String>(); // for (Map<String,String> row : csvIteratorExistingHeaders(directory + "/xustudy/newmapping.csv")) { for (Map<String, String> row : csvIteratorExistingHeaders(directory + "/summerschool/newmapping.csv")) { xu_forumname2forum.put(row.get("forumname"), row.get("forumid")); } File[] listOfFiles = new File(directory + "/summerschool/chats/").listFiles(); // File[] listOfFiles = new File(directory + "/xustudy/chatlogs_transactivity_annotated/").listFiles(); for (File file : listOfFiles) { if (file.isFile() && file.getName().endsWith(".csv")) { //if (true) break; String n = file.getName(); String forum_id = "", team_id = ""; if (n.startsWith("mturkno")) { forum_id = n.substring(7, n.length() - 5); } else if (n.startsWith("mturk")) { forum_id = n.substring(5, n.length() - 5); } forum_id = xu_forumname2forum.getOrDefault(forum_id, "0"); team_id = n.substring(n.length() - 5, n.length() - 4); if (!forum_id.equals("0")) { int lineno = 0; if (summerSchool || n.startsWith("mturkno")) { /* ,type,username,useraddress,userid,timestamp,roomname,content,neg, * 1,presence,BazaarAgent,128.2.220.133:35582,N,6/4/16 21:24,mturkno798238,join,bazaar, */ for (Map<String, String> row : csvIteratorExistingHeaders(file.getAbsolutePath())) { if (row.get("type") == "presence") { //mcs.mapChatInteraction(row.get("timestamp") + ":00", forum_id + ":" + row.get("username"), forum_id, team_id, row.get("content"), // xuDiscourseName, datasetName, "chats/" + file.getName(), "lineno", lineno); } else if (row.get("username") != null && row.get("username").length() > 0) { mcs.mapChat(row.get("timestamp") + ":00", forum_id + ":" + row.get("username"), forum_id, team_id, row.get("content"), xuDiscourseName, datasetName, "chats/" + file.getName(), "lineno", Long.toString(lineno)); } lineno += 1; } } else { /* * 7/11/16,20:53:59,0,Andy,1.46828E+12,Hi,neg,neg,,, * 7/11/16,20:54:07,0,UKCats,1.46828E+12,Hi all,neg,neg,,, */ System.out.println("Trying to scan " + file.getAbsolutePath()); for (Map<String, String> row : csvIteratorNoHeaders(file.getAbsolutePath(), "date,time,zero,username,number,content,fld1,transactivity,fld3,fld4,fld5,ign1,ign2,ign3,ign4,ign5,ign6")) { if (row.get("username") != null && row.get("username").length() > 0) { mcs.mapChat(row.get("date") + " " + row.get("time"), forum_id + ":" + row.get("username"), forum_id, team_id, row.get("content"), xuDiscourseName, datasetName, "chats/" + file.getName(), "lineno", Long.toString(lineno)); } lineno += 1; } } } else { System.out.println("Chat session " + file.getName() + " can't be identified"); } } } /*HOW: * Read xu/userid-namemap -> to get username to userid-within-group * Read xu/newmapping -> to get forumid -> groupname * Read xu/chatlogs -> * add dp for each file, link to experiment, group, team * for each posting add user, text, date; link to dp * */ System.out.println("Doing pre/post tests"); Pattern forum_team_user = Pattern.compile("(\\d\\d\\d)_(\\d+)_(\\d+)"); Matcher m1 = forum_team_user.matcher("234pre234_2_3.csv"); m1.find(); assert m1.group(1) == "234"; //Iterator<File> it = FileUtils.iterateFiles(new File(directory + "/xustudy/preposttest"), null, true); Iterator<File> it = FileUtils.iterateFiles(new File(directory + "/summerschool/242_pretest"), null, true); while (it.hasNext()) { File test = it.next(); if (test.isFile() && test.getName().endsWith(".csv")) { System.out.println("Doing test " + test.getName()); String n = test.getName(); Matcher m = forum_team_user.matcher(n); if (m.find()) { String forum_id = m.group(1); String team_id = m.group(2); String user_id = m.group(3); String testtype = "Pretest"; if (n.contains("post")) { testtype = "Posttest"; } String content = FileUtils.readFileToString(test); content = content.substring(content.indexOf("\n")); // skip first line, which is a false csv header String xu_id = forum_id + "_" + team_id + "_" + user_id; String username = xu_id2username.get(xu_id); System.out.println("Scanning " + testtype + " " + n + " by " + username + " on team " + forum_id + "_" + team_id); mcs.mapFile(forum_id, team_id, username, testtype + " by " + username, testtype.equals("Posttest") ? ContributionTypes.POSTTEST : ContributionTypes.PRETEST, content, xuDiscourseName, datasetName, "preposttests", "for_user", xu_id); } } } System.out.println("Doing xu proposals"); //Iterable<File> it2 = () -> FileUtils.iterateFiles(new File(directory + "/xustudy/group_proposals_txt/"), null, false); Iterable<File> it2 = () -> FileUtils.iterateFiles(new File(directory + "/summerschool/proposals/"), null, false); for (File prop : it2) { //if (true) break; if (prop.isFile() && prop.getName().endsWith(".txt")) { System.out.println("Doing proposal " + prop.getName()); String n = prop.getName(); String forum_id = "", team_id = ""; forum_id = n.substring(0, n.length() - 5); team_id = n.substring(n.length() - 5, n.length() - 4); forum_id = xu_forumname2forum.getOrDefault(forum_id, "0"); System.out.println("Scanning proposal " + n + " by " + forum_id + "_" + team_id); String content = FileUtils.readFileToString(prop); mcs.mapFile(forum_id, team_id, forum_id + "_" + team_id, "Proposal by " + forum_id + "_" + team_id, ContributionTypes.PROPOSAL, content, xuDiscourseName, datasetName, "proposals", "for_team", forum_id + "_" + team_id); } } if (!summerSchool) { System.out.println("Doing wen proposals"); Iterable<File> it3 = () -> FileUtils.iterateFiles(new File(directory + "/wenstudy/proposals/"), null, false); for (File prop : it3) { //if (true) break; if (prop.isFile() && prop.getName().endsWith(".csv")) { System.out.println("Doing proposal " + prop.getName()); String n = prop.getName(); String group_id = "", team_id = ""; group_id = wenGroup(n.substring(0, n.length() - 4)); team_id = wenTeam(n.substring(0, n.length() - 4)); System.out.println("Scanning proposal " + n + " by " + group_id + "_" + team_id + ": " + prop.getAbsolutePath()); // This kludgey code handles the fact that: // * These look like csv files, but quoted strings contain unescaped quotes // * Sometimes there are multiple columns, sometimes not, but we only care about the first column // * Usually if there are extra columns, the first few are empty, so we can ignore anything after ,, // * First row is column names, but first column name is usually (not always) blank String[] content2 = FileUtils.readFileToString(prop).split("\n"); content2[0] = ""; // Skip header row String content = ""; for (String row : content2) { String keep = ""; if (row.length() >= 2 && row.startsWith("\"")) { keep += row.substring(1, row.length() - 1); } else { keep += row + "\n"; } content += keep.split(",,")[0]; // Sometimes these have multiple rows and we only care about the first column } mcs.mapFile(group_id, team_id, group_id + "_" + team_id, "Proposal by " + group_id + "_" + team_id, ContributionTypes.PROPOSAL, content, wenDiscourseName, datasetName, "proposals", "for_team", group_id + "_" + team_id); } } System.out.println("Doing wen study chats"); File[] listOfFiles2 = new File(directory + "/wenstudy/chats/").listFiles(); for (File file : listOfFiles2) { if (file.isFile() && file.getName().endsWith(".csv")) { //if (true) break; String n = file.getName(); String group_id = "", team_id = ""; group_id = wenGroup(n.substring(0, n.length() - 4)); team_id = wenTeam(n.substring(0, n.length() - 4)); int lineno = 0; /* userid,chat,, * asdf,"Plan 2 exceeds the total budget, though.",, * asdf,"Plan 1 is the only one that falls within their ""tight"" budget.",1, */ for (Map<String, String> row : csvIteratorExistingHeaders(file.getAbsolutePath())) { System.out.println(group_id + team_id + " " + row.get("userid") + " says " + row.get("chat").substring(0, java.lang.Math.min(30, row.get("chat").length()))); if (row.get("userid") != null && row.get("userid").length() > 0 && row.get("userid").length() < 50) { mcs.mapChat(null, group_id + ":" + row.get("userid"), group_id, team_id, row.get("chat"), wenDiscourseName, datasetName, "chats/" + file.getName(), "lineno", Long.toString(lineno)); } lineno += 1; } } } } /* * read xu/preposttest files -> * look up author from * create dp for pre/post+teamname, dpr link to team, group, experiment from filename * import as contribution/content; ignore first line * read xu/proposal files * Leave author blank; place directly as posting under team's dp * read wen/chats * make dp for each file: wen team ff1 chat, link to team, group, experiment * put contribution from each one. Number sequentially from 1972-01-01 incrementing by one minute each through the whole set of files. * read wen/proposals * leave author blank; place directly as posting under team's dp. title="wen team ff1 proposal" Mechanism: * thingy to read in a csv file with or without a fixed set of fields * thingy to coalesce a whole csv column into a single string with carriage returns * thingy to store a map * thingy to write elements and store the discoursedb-indexes to a map Map<String, String> roomIdNameMap = new HashMap<>(); List<String> messages = new ArrayList<>(); //Read input file and preprocess String lineFragment = null; for(String line:FileUtils.readLines(new File(messageFileDir))){ //line fragments occur in case we have line feeds in a column if(lineFragment!=null){ line=lineFragment+line; lineFragment=null; } if (line.endsWith("\\")||line.endsWith("\\\r\f")){ line = line.replaceAll("\\\r\f", ""); lineFragment = line; }else{ if (line.contains("\\\"We're Ready\\\"")) { line = line.replaceAll("\"We're Ready\\\\\"", "We're Ready\\\\"); } if (line.contains("\\\"ready\\\"")) { line = line.replaceAll("\\\\\"ready\\\\\"", "\\\\ready\\\\"); } if (line.contains("\\\""+agentname+"\\\"")){ line = line.replaceAll("\\\\\""+agentname+"\\\\\"", "\\\\"+agentname+"\\\\"); } messages.add(line); } } // Phase 1: read through input room file once and map all entities try (InputStream in = new FileInputStream(roomFileDir)) { CsvMapper mapper = new CsvMapper(); CsvSchema schema = mapper.schemaFor(Room.class).withColumnSeparator(','); MappingIterator<Room> rIter = mapper.readerFor(Room.class).with(schema).readValues(in); while (rIter.hasNextValue()) { Room r = rIter.next(); if (!roomIdNameMap.containsKey(r.getId())) roomIdNameMap.put(r.getId(), r.getName()); converterService.mapRoom(r, dataSetName, discourseName); } } catch (IOException e) { log.error("Error reading room file",e); } // Phase 2: read through input message file and map relationships between room and message CsvMapper mapper = new CsvMapper(); CsvSchema schema = mapper.schemaFor(Message.class).withColumnSeparator(','); for(String message:messages){ Message m = mapper.readerFor(Message.class).with(schema).readValue(message); if (m.getType().equals("text") || m.getType().equals("image") || m.getType().equals("private")){ converterService.mapMessage(m, dataSetName, discourseName, roomIdNameMap); }else{ converterService.mapInteraction(m, dataSetName, discourseName, roomIdNameMap); } } */ }
From source file:fr.inria.soctrace.tools.ocelotl.core.caches.DataCache.java
/** * Load the existing cache files from the current cache directory *///from ww w . jav a2 s .c o m private void readCachedData() { File workDir = new File(cacheDirectory); // Clear the current cache files cachedData.clear(); if (workDir.exists()) { Iterator<File> anIT = FileUtils.iterateFiles(workDir, null, true); while (anIT.hasNext()) { File traceCache = anIT.next(); // Check that it is a cache file if (!traceCache.getName().endsWith(OcelotlConstants.DataCacheSuffix)) continue; // Try parsing the file and get the cache parameters CacheParameters param = parseTraceCache(traceCache); // If parsing was successful if (param.getTraceID() != -1) { // Register the cache file cachedData.put(param, traceCache); logger.debug("Found " + param.getTraceName() + " in " + traceCache.toString() + ", " + param.getMicroModelType() + ", " + param.getVisuAggOperator() + ", " + param.getStartTimestamp() + ", " + param.getEndTimestamp()); } } computeCacheSize(); } else { System.err.println("The provided cache directory (" + cacheDirectory + ") does not exist"); } }
From source file:es.ehu.si.ixa.qwn.ppv.CLI.java
public final void eval() throws IOException { String corpus = parsedArguments.getString("corpus"); String test = parsedArguments.getString("testfile"); String lexicon = parsedArguments.getString("lexicon"); String estimator = parsedArguments.getString("estimator"); String synset = parsedArguments.getString("synset"); boolean shouldOptimize = parsedArguments.getBoolean("optimize"); boolean weights = parsedArguments.getBoolean("weights"); float thres = Float.parseFloat(parsedArguments.getString("threshold")); float lexThres = Float.parseFloat(parsedArguments.getString("lexScoreThreshold")); // parameter control. if (synset.equals("mfs")) { System.err.println(/*from w ww. j a v a 2s. com*/ "\nWARNING: You have selected to check polarity based on the most frequent sense (MFS). " + "Take into account that MFS works as if it were the first sense. It is left up to the " + "user to provide a corpus which has the MFS as the first sense in the annotated corpus"); } // parameter combination control. Mohammad's evaluation is always done without optimization. if (shouldOptimize && estimator.equals("moh")) { System.err.println( "\nWARNING: Mohammad's (2009) polarity score is directly applied over the given corpus, " + "it does not require optimization. Thus, optimization WILL NOT BE PERFORMED," + "and testfile WILL NOT BE USED."); shouldOptimize = false; } // parameter combination control. Mohammad's evaluation threshold is always 0. if (thres != 0 && estimator.equals("moh")) { System.err.println("\nWARNING: Mohammad's (2009) polarity score needs no threshold." + " It WILL NOT BE taken into account."); thres = 0; } System.out.println("QWN-PPV: Lexicon evaluator: "); File lexFile = new File(lexicon); // Given lexicon path does not exist, print error and exit; if (!lexFile.exists()) { System.out.println("QWN-PPV: Given lexicon does not exist. Evaluation can not continue.\n"); System.exit(1); } // Given lexicon path is a single file; else if (lexFile.isFile()) { System.out.println("\tcorpus: " + corpus + "\n\tlexicon: " + lexFile.getAbsolutePath() + "\n\tSense information: " + synset + "\n"); System.err.println("\tcorpus: " + corpus + "\n\tlexicon: " + lexFile.getAbsolutePath() + "\n\tSense information: " + synset + "\n"); //load Lexicon Lexicon lex = new Lexicon(lexFile, synset, lexThres); //create evaluator object Evaluator evalCorpus = new Evaluator(lex, synset, estimator); evalCorpus.setThreshold(thres); Map<String, Float> results = evalCorpus.processCorpus(corpus, shouldOptimize, weights); if (shouldOptimize && test != "") { System.out.println("optimization finished, starting test"); evalCorpus.setThreshold(results.get("thresh")); results = evalCorpus.processCorpus(test, false, weights); } //System.out.println("eval avg done"+results.toString()); System.out.println("\tThreshold = " + results.get("thresh") + "\n\t" + results.get("predPos") + " labeled as positive\t" + results.get("predNeg") + " labeled as negative\t " + results.get("undefined") + " undefined. \n" + "\tAccuracy => " + results.get("Acc") + "\n\tPositive docs: P => " + results.get("Ppos") + "\tR => " + results.get("Rpos") + "\tF => " + results.get("Fpos") + "\n" + "\tNegative docs: P => " + results.get("Pneg") + "\tR => " + results.get("Rneg") + "\tF => " + results.get("Fneg") + " \n"); } // Given lexicon path is a directory; else if (lexFile.isDirectory()) { //get all files with the *.dict extension and evaluate them) String[] allowedDictExtensions = { "dict" }; Iterator<File> it = FileUtils.iterateFiles(lexFile, allowedDictExtensions, false); while (it.hasNext()) { File itLex = it.next(); System.out.println("\tcorpus: " + corpus + "\n\tlexicon: " + itLex.getAbsolutePath() + "\n\tSense information: " + synset + "\n"); System.err.println("\tcorpus: " + corpus + "\n\tlexicon: " + itLex.getAbsolutePath() + "\n\tSense information: " + synset + "\n"); //load Lexicon Lexicon lex = new Lexicon(itLex, synset, lexThres); //create evaluator object Evaluator evalCorpus = new Evaluator(lex, synset, estimator); evalCorpus.setThreshold(thres); Map<String, Float> results = evalCorpus.processCorpus(corpus, shouldOptimize, weights); if (shouldOptimize && test != "") { System.out.println("optimization finished, starting test"); evalCorpus.setThreshold(results.get("thresh")); results = evalCorpus.processCorpus(test, false, weights); } //System.out.println("eval avg done"+results.toString()); System.out.println("\tThreshold = " + results.get("thresh") + "\n\t" + results.get("predPos") + " labeled as positive\t" + results.get("predNeg") + " labeled as negative\t " + results.get("undefined") + " undefined. \n" + "\tAccuracy => " + results.get("Acc") + "\n\tPositive docs: P => " + results.get("Ppos") + "\tR => " + results.get("Rpos") + "\tF => " + results.get("Fpos") + "\n" + "\tNegative docs: P => " + results.get("Pneg") + "\tR => " + results.get("Rneg") + "\tF => " + results.get("Fneg") + " \n"); } } System.out.println("QWN-PPV: Lexicon evaluator: End."); }
From source file:com.ibm.soatf.component.ftp.FTPComponent.java
public File getFile(File workingDirectory, String fileName, String fileContent) throws FtpComponentException { String pattern = "*_" + fileName; Iterator<File> it = FileUtils.iterateFiles(workingDirectory, new WildcardFileFilter(pattern), TrueFileFilter.INSTANCE);/*w w w . j a v a2 s . co m*/ int count = 0; File f = null; while (it.hasNext()) { ++count; f = it.next(); } if (count == 0) { f = generateFile(workingDirectory, fileName, fileContent); } if (count > 1) { //TODO } return f; }