List of usage examples for java.io PrintStream close
public void close()
From source file:com.atlassian.jira.webtests.JIRAWebTest.java
public void dumpScreen(String filename) { try {//from w ww. ja va2 s . c o m PrintStream printStream = new PrintStream(new FileOutputStream( new File(getEnvironmentData().getWorkingDirectory().getAbsolutePath() + FS + filename + HTM))); dumpResponse(printStream); printStream.close(); } catch (FileNotFoundException e) { e.printStackTrace(); } }
From source file:com.rapid.core.Application.java
public void initialise(ServletContext servletContext, boolean createResources) throws JSONException, InstantiationException, IllegalAccessException, ClassNotFoundException, IllegalArgumentException, InvocationTargetException, SecurityException, NoSuchMethodException, IOException { // get the logger Logger logger = (Logger) servletContext.getAttribute("logger"); // trace log that we're initialising logger.trace("Initialising application " + _name + "/" + _version); // initialise the security adapter setSecurityAdapter(servletContext, _securityAdapterType); // initialise the form adapter setFormAdapter(servletContext, _formAdapterType); // initialise the resource includes collection _resources = new Resources(); // if there is any app JavaScript functions - this is for backwards compatibility as _functions have been moved to JavaScript resources if (_functions != null) { // initialise app resources if need be if (_appResources == null) _appResources = new Resources(); // add _functions as JavaScript resource to top of list _appResources.add(0, new Resource("Application functions", 1, _functions)); // remove the _functions _functions = null;//from w w w. j a va 2 s .co m } // if the created date is null set to today if (_createdDate == null) _createdDate = new Date(); // when importing an application we need to initialise but don't want the resource folders made in the old applications name if (createResources) { // get the jsonControls JSONArray jsonControls = (JSONArray) servletContext.getAttribute("jsonControls"); // get the jsonActions JSONArray jsonActions = (JSONArray) servletContext.getAttribute("jsonActions"); // string builders for the different sections in our rapid.js file StringBuilder resourceJS = new StringBuilder(); StringBuilder initJS = new StringBuilder(); StringBuilder dataJS = new StringBuilder(); StringBuilder actionJS = new StringBuilder(); // string builder for our rapid.css file StringBuilder resourceCSS = new StringBuilder(); // check controls if (jsonControls != null) { // check control types if (_controlTypes != null) { // remove the page control (if it's there) _controlTypes.remove("page"); // add it to the top of the list _controlTypes.add(0, "page"); // collection of dependent controls that need adding ArrayList<String> dependentControls = new ArrayList<String>(); // loop control types used by this application for (String controlType : _controlTypes) { // loop all available controls for (int i = 0; i < jsonControls.length(); i++) { // get the control JSONObject jsonControl = jsonControls.getJSONObject(i); // check if we're on the type we need if (controlType.equals(jsonControl.optString("type"))) { // look for any dependent control types JSONObject dependantTypes = jsonControl.optJSONObject("dependentTypes"); // if we got some if (dependantTypes != null) { // look for an array JSONArray dependantTypesArray = dependantTypes.optJSONArray("dependentType"); // if we got one if (dependantTypesArray != null) { // loop the array for (int j = 0; j < dependantTypesArray.length(); j++) { String dependantType = dependantTypesArray.getString(j); if (!_controlTypes.contains(dependantType) && !dependentControls.contains(dependantType)) dependentControls.add(dependantType); } } else { // just use the object String dependantType = dependantTypes.getString("dependentType"); if (!_controlTypes.contains(dependantType) && !dependentControls.contains(dependantType)) dependentControls.add(dependantType); } } // we're done break; } // available control type check } // available control types loop } // application control types loop // now add all of the dependent controls _controlTypes.addAll(dependentControls); // loop control types used by this application for (String controlType : _controlTypes) { // loop all available controls for (int i = 0; i < jsonControls.length(); i++) { // get the control JSONObject jsonControl = jsonControls.getJSONObject(i); // check if we're on the type we need if (controlType.equals(jsonControl.optString("type"))) { // add any resources (actions can have them too) addResources(jsonControl, "control", resourceJS, resourceCSS); // get any initJavaScript String js = jsonControl.optString("initJavaScript", ""); // check we got some if (js.length() > 0) { initJS.append("\nfunction Init_" + jsonControl.getString("type") + "(id, details) {\n"); initJS.append(" " + js.trim().replace("\n", "\n ")); initJS.append("\n}\n"); } // check for a getData method String getDataFunction = jsonControl.optString("getDataFunction"); // if there was something if (getDataFunction != null) { // clean and print! (if not an empty string) if (getDataFunction.trim().length() > 0) dataJS.append("\nfunction getData_" + controlType + "(ev, id, field, details) {\n " + getDataFunction.trim().replace("\n", "\n ") + "\n}\n"); } // check for a setData method String setDataFunction = jsonControl.optString("setDataJavaScript"); // if there was something if (setDataFunction != null) { // clean and print! (if not an empty string) if (setDataFunction.trim().length() > 0) dataJS.append("\nfunction setData_" + controlType + "(ev, id, field, details, data, changeEvents) {\n " + setDataFunction.trim().replace("\n", "\n ") + "\n}\n"); } // retrieve any runtimeProperties JSONObject jsonRuntimePropertyCollection = jsonControl .optJSONObject("runtimeProperties"); // check we got some if (jsonRuntimePropertyCollection != null) { // get the first one JSONObject jsonRuntimeProperty = jsonRuntimePropertyCollection .optJSONObject("runtimeProperty"); // get an array JSONArray jsonRunTimeProperties = jsonRuntimePropertyCollection .optJSONArray("runtimeProperty"); // initialise counters int index = 0; int count = 0; // if we got an array if (jsonRunTimeProperties != null) { // retain the first entry in the object jsonRuntimeProperty = jsonRunTimeProperties.getJSONObject(0); // retain the size count = jsonRunTimeProperties.length(); } do { // get the type String type = jsonRuntimeProperty.getString("type"); // get the get function String getFunction = jsonRuntimeProperty.optString("getPropertyFunction", null); // print the get function if there was one if (getFunction != null) dataJS.append("\nfunction getProperty_" + controlType + "_" + type + "(ev, id, field, details) {\n " + getFunction.trim().replace("\n", "\n ") + "\n}\n"); // get the set function String setFunction = jsonRuntimeProperty.optString("setPropertyJavaScript", null); // print the get function if there was one if (setFunction != null) dataJS.append("\nfunction setProperty_" + controlType + "_" + type + "(ev, id, field, details, data, changeEvents) {\n " + setFunction.trim().replace("\n", "\n ") + "\n}\n"); // increment index index++; // get the next one if (index < count) jsonRuntimeProperty = jsonRunTimeProperties.getJSONObject(index); } while (index < count); } // we're done with this jsonControl break; } } // jsonControls loop } // control types loop } // control types check } // jsonControls check // check actions if (jsonActions != null) { // check action types if (_actionTypes != null) { // collection of dependent controls that need adding ArrayList<String> dependentActions = new ArrayList<String>(); // loop control types used by this application for (String actionType : _actionTypes) { // loop all available controls for (int i = 0; i < jsonActions.length(); i++) { // get the action JSONObject jsonAction = jsonActions.getJSONObject(i); // check if we're on the type we need if (actionType.equals(jsonAction.optString("type"))) { // look for any dependant control types JSONObject dependantTypes = jsonAction.optJSONObject("dependentTypes"); // if we got some if (dependantTypes != null) { // look for an array JSONArray dependantTypesArray = dependantTypes.optJSONArray("dependentType"); // if we got one if (dependantTypesArray != null) { // loop the array for (int j = 0; j < dependantTypesArray.length(); j++) { String dependantType = dependantTypesArray.getString(j); if (!_actionTypes.contains(dependantType) && !dependentActions.contains(dependantType)) dependentActions.add(dependantType); } } else { // just use the object String dependantType = dependantTypes.getString("dependentType"); if (!_actionTypes.contains(dependantType) && !dependentActions.contains(dependantType)) dependentActions.add(dependantType); } } // we're done break; } } } // now add all of the dependent controls _controlTypes.addAll(dependentActions); // loop action types used by this application for (String actionType : _actionTypes) { // loop jsonActions for (int i = 0; i < jsonActions.length(); i++) { // get action JSONObject jsonAction = jsonActions.getJSONObject(i); // check the action is the one we want if (actionType.equals(jsonAction.optString("type"))) { // add any resources (controls can have them too) addResources(jsonAction, "action", resourceJS, resourceCSS); // get action JavaScript String js = jsonAction.optString("actionJavaScript"); // only produce rapid action is this is rapid app if (js != null && ("rapid".equals(_id) || !"rapid".equals(actionType))) { // clean and print! (if not an empty string) if (js.trim().length() > 0) actionJS.append("\n" + js.trim() + "\n"); } // move onto the next action type break; } } // jsonActions loop } // action types loop } // action types check } // jsonAction check // assume no theme css String themeCSS = null; // assume no theme name String themeName = null; // check the theme type if (_themeType != null) { // get the themes List<Theme> themes = (List<Theme>) servletContext.getAttribute("themes"); // check we got some if (themes != null) { // loop them for (Theme theme : themes) { // check type if (_themeType.equals(theme.getType())) { // retain the theme CSS themeCSS = theme.getCSS(); // retain the name themeName = theme.getName(); // get any resources addResources(theme.getResources(), "theme", themeName, null, resourceJS, resourceCSS); // we're done break; } } } } // put the appResources at the end so they can be overrides if (_appResources != null) { for (Resource resource : _appResources) { // create new resource based on this one (so that the dependancy doesn't get written back to the application.xml file) Resource appResource = new Resource(resource.getType(), resource.getContent(), ResourceDependency.RAPID); // if the type is a file or link prefix with the application folder switch (resource.getType()) { case Resource.JAVASCRIPTFILE: case Resource.CSSFILE: // files are available on the local file system so we prefix with the webfolder appResource.setContent(getWebFolder(this) + (resource.getContent().startsWith("/") ? "" : "/") + resource.getContent()); break; case Resource.JAVASCRIPTLINK: case Resource.CSSLINK: // links are not so go in as-is appResource.setContent(resource.getContent()); break; } // add new resource based on this one but with Rapid dependency _resources.add(appResource); } } // create folders to write the rapid.js file String applicationPath = getWebFolder(servletContext); File applicationFolder = new File(applicationPath); if (!applicationFolder.exists()) applicationFolder.mkdirs(); // write the rapid.js file FileOutputStream fos = new FileOutputStream(applicationPath + "/rapid.js"); PrintStream ps = new PrintStream(fos); // write the rapid.min.js file FileOutputStream fosMin = new FileOutputStream(applicationPath + "/rapid.min.js"); PrintWriter pw = new PrintWriter(fosMin); // file header ps.print( "\n/* This file is auto-generated on application load and save - it is minified when the application status is live */\n"); // check functions if (_functions != null) { if (_functions.length() > 0) { // header (this is removed by minify) ps.print("\n\n/* Application functions JavaScript */\n\n"); // insert params String functionsParamsInserted = insertParameters(servletContext, _functions); // print ps.print(functionsParamsInserted); // print minify Minify.toWriter(functionsParamsInserted, pw, Minify.JAVASCRIPT); } } // check resource js if (resourceJS.length() > 0) { // header ps.print("\n\n/* Control and Action resource JavaScript */\n\n"); // insert params String resourceJSParamsInserted = insertParameters(servletContext, resourceJS.toString()); // print ps.print(resourceJS.toString()); // print minify Minify.toWriter(resourceJSParamsInserted, pw, Minify.JAVASCRIPT); } // check init js if (initJS.length() > 0) { // header ps.print("\n\n/* Control initialisation methods */\n\n"); // insert params String initJSParamsInserted = insertParameters(servletContext, initJS.toString()); // print ps.print(initJS.toString()); // print minify Minify.toWriter(initJSParamsInserted, pw, Minify.JAVASCRIPT); } // check datajs if (dataJS.length() > 0) { // header ps.print("\n\n/* Control getData and setData methods */\n\n"); // insert params String dataJSParamsInserted = insertParameters(servletContext, dataJS.toString()); // print ps.print(dataJS.toString()); // print minify Minify.toWriter(dataJSParamsInserted, pw, Minify.JAVASCRIPT); } // check action js if (actionJS.length() > 0) { // header ps.print("\n\n/* Action methods */\n\n"); // insert params String actionParamsInserted = insertParameters(servletContext, actionJS.toString()); // print ps.print(actionJS.toString()); // print minify Minify.toWriter(actionParamsInserted, pw, Minify.JAVASCRIPT); } // close debug writer and stream ps.close(); fos.close(); // close min writer and stream pw.close(); fosMin.close(); // get the rapid CSS into a string and insert parameters String resourceCSSWithParams = insertParameters(servletContext, resourceCSS.toString()); String appThemeCSSWithParams = insertParameters(servletContext, themeCSS); String appCSSWithParams = insertParameters(servletContext, _styles); // write the rapid.css file fos = new FileOutputStream(applicationPath + "/rapid.css"); ps = new PrintStream(fos); ps.print( "\n/* This file is auto-generated on application load and save - it is minified when the application status is live */\n\n"); if (resourceCSSWithParams != null) { ps.print(resourceCSSWithParams.trim()); } if (appThemeCSSWithParams != null) { ps.print("\n\n/* " + themeName + " theme styles */\n\n"); ps.print(appThemeCSSWithParams.trim()); } if (appCSSWithParams != null) { ps.print("\n\n/* Application styles */\n\n"); ps.print(appCSSWithParams.trim()); } ps.close(); fos.close(); // minify it to a rapid.min.css file Minify.toFile(resourceCSSWithParams + appCSSWithParams, applicationPath + "/rapid.min.css", Minify.CSS); // check the status if (_status == STATUS_LIVE) { // add the application js min file as a resource _resources.add(new Resource(Resource.JAVASCRIPTFILE, getWebFolder(this) + "/rapid.min.js", ResourceDependency.RAPID)); // add the application css min file as a resource _resources.add(new Resource(Resource.CSSFILE, getWebFolder(this) + "/rapid.min.css", ResourceDependency.RAPID)); } else { // add the application js file as a resource _resources.add(new Resource(Resource.JAVASCRIPTFILE, getWebFolder(this) + "/rapid.js", ResourceDependency.RAPID)); // add the application css file as a resource _resources.add(new Resource(Resource.CSSFILE, getWebFolder(this) + "/rapid.css", ResourceDependency.RAPID)); } // loop all resources and minify js and css files for (Resource resource : _resources) { // get the content (which is the filename) String fileName = resource.getContent(); // only interested in js and css files switch (resource.getType()) { case Resource.JAVASCRIPTFILE: // get a file for this File jsFile = new File( servletContext.getRealPath("/") + (fileName.startsWith("/") ? "" : "/") + fileName); // if the file exists, and it's in the scripts folder and ends with .js if (jsFile.exists() && fileName.startsWith("scripts/") && fileName.endsWith(".js")) { // derive the min file name by modifying the start and end String fileNameMin = "scripts_min/" + fileName.substring(8, fileName.length() - 3) + ".min.js"; // get a file for minifying File jsFileMin = new File(servletContext.getRealPath("/") + "/" + fileNameMin); // if this file does not exist if (!jsFileMin.exists()) { // make any dirs it may need jsFileMin.getParentFile().mkdirs(); // minify to it Minify.toFile(jsFile, jsFileMin, Minify.JAVASCRIPT); } // if this application is live, update the resource to the min file if (_status == STATUS_LIVE) resource.setContent(fileNameMin); } break; case Resource.CSSFILE: // get a file for this File cssFile = new File( servletContext.getRealPath("/") + (fileName.startsWith("/") ? "" : "/") + fileName); // if the file exists, and it's in the scripts folder and ends with .js if (cssFile.exists() && fileName.startsWith("styles/") && fileName.endsWith(".css")) { // derive the min file name by modifying the start and end String fileNameMin = "styles_min/" + fileName.substring(7, fileName.length() - 4) + ".min.css"; // get a file for minifying File cssFileMin = new File(servletContext.getRealPath("/") + "/" + fileNameMin); // if this file does not exist if (!cssFileMin.exists()) { // make any dirs it may need cssFileMin.getParentFile().mkdirs(); // minify to it Minify.toFile(cssFile, cssFileMin, Minify.CSS); } // if this application is live, update the resource to the min file if (_status == STATUS_LIVE) resource.setContent(fileNameMin); } break; } } // loop resources // a list for all of the style classes we're going to send up with _styleClasses = new ArrayList<String>(); // populate the list of style classes by scanning the global styles scanStyleClasses(_styles, _styleClasses); // and any theme scanStyleClasses(appThemeCSSWithParams, _styleClasses); // remove any that have the same name as controls if (jsonControls != null) { // loop them for (int i = 0; i < jsonControls.length(); i++) { // remove any classes with this controls type _styleClasses.remove(jsonControls.getJSONObject(i).getString("type")); } } } // create resources // empty the list of page variables so it's regenerated _pageVariables = null; // debug log that we initialised logger.debug( "Initialised application " + _name + "/" + _version + (createResources ? "" : " (no resources)")); }
From source file:ca.uvic.cs.tagsea.statistics.svn.jobs.SVNCommentScanningJob.java
protected IStatus run(IProgressMonitor monitor) { IProject[] projects = ResourcesPlugin.getWorkspace().getRoot().getProjects(); ISVNClientAdapter client;/*from www . ja va 2 s . c o m*/ List svnResources = new LinkedList(); for (int i = 0; i < projects.length; i++) { IProject project = projects[i]; ISVNLocalResource r = SVNWorkspaceRoot.getSVNResourceFor(project); try { if (r != null && r.isManaged()) { svnResources.add(r); } } catch (SVNException e) { //do nothing, continue t the next } } monitor.beginTask("Scanning subversion projects...", svnResources.size() * 1000000); IPath state = SVNStatistics.getDefault().getStateLocation(); File tempdir = state.append("temp").toFile(); if (!tempdir.isDirectory()) { if (tempdir.exists()) { tempdir.delete(); } tempdir.mkdir(); } deleteTemps(tempdir); for (Iterator it = svnResources.iterator(); it.hasNext();) { ISVNLocalResource svnProject = (ISVNLocalResource) it.next(); //used to make sure that we don't repeat old comments. HashMap fileCommentsMap = new HashMap(); fileEntryMap = new HashMap(); monitor.subTask("Getting project information for " + svnProject.getName()); //create a temp file for each project. They will be uploaded //to the server. String projectName = svnProject.getName(); //names are guaranteed unique try { ISVNRemoteResource remote = null; for (int tries = 0; remote == null && tries < 10; tries++) { try { remote = svnProject.getLatestRemoteResource(); } catch (Exception e) { } ; if (remote == null) { SVNStatistics.getDefault().getLog().log(new Status(IStatus.WARNING, SVNStatistics.PLUGIN_ID, IStatus.WARNING, "could not get remote resource for " + svnProject.getName() + "... trying again.", null)); try { // @tag tagsea.bug.subclipse : it seems that sublcipse has a synchronization problem. Wait a little while and try again. Thread.sleep(1000); } catch (InterruptedException e1) { return new Status(IStatus.ERROR, SVNProviderPlugin.ID, IStatus.ERROR, "Could not communicate with remote resource.", null); } } } if (remote == null) { SVNStatistics.getDefault().getLog().log(new Status(IStatus.ERROR, SVNStatistics.PLUGIN_ID, 0, "Could not get a remote resource", null)); monitor.worked(1000000); continue; } ISVNRepositoryLocation repository = remote.getRepository(); client = repository.getSVNClient(); // @tag tagsea.statistics.enhance : It seems best to use this password callback because that way, the passwords can be saved with the workspace. But, it might be confusing to see for the first time. client.addPasswordCallback(SVNProviderPlugin.getPlugin().getSvnPromptUserPassword()); SVNRevision.Number revision = remote.getLastChangedRevision(); long revNum = revision.getNumber(); int revisionWork = 1000000; ILogEntry[] entries; try { entries = remote.getLogEntries(new NullProgressMonitor()); } catch (TeamException e1) { monitor.worked(revisionWork); e1.printStackTrace(); continue; } if (revNum > 0) { revisionWork = 1000000 / (int) revNum; } for (int ei = 0; ei < entries.length; ei++) { ILogEntry entry = entries[ei]; revision = entry.getRevision(); File tempFile = state.append( projectName + "." + getDateString() + "." + revision.getNumber() + ".comments.txt") .toFile(); if (tempFile.exists()) { tempFile.delete(); } try { tempFile.createNewFile(); } catch (IOException e) { //skip to the next one. continue; } PrintStream out; try { out = new PrintStream(tempFile); } catch (IOException e) { continue; } out.println(remote.getUrl() + " Revision:" + revision.getNumber()); monitor.subTask("Finding java resources: " + svnProject.getName() + "..."); SubProgressMonitor revMonitor = new SubProgressMonitor(monitor, revisionWork); if (monitor.isCanceled()) { return Status.CANCEL_STATUS; } monitor.subTask("temporarily checking out " + svnProject.getName() + "..."); SubProgressMonitor subPm = new SubProgressMonitor(revMonitor, 10); try { OperationManager.getInstance().beginOperation(client, new OperationProgressNotifyListener(subPm)); client.checkout(remote.getUrl(), new File(tempdir, svnProject.getName()), revision, true); } catch (SVNClientException e) { //I wish that there were a better way to do this, but it seem that we //have to just keep decrementing it. revMonitor.done(); revNum--; revision = new SVNRevision.Number(revNum); continue; } finally { OperationManager.getInstance().endOperation(); subPm.done(); } if (monitor.isCanceled()) { return Status.CANCEL_STATUS; } List files = findJavaFiles(tempdir); int work = 0; if (files.size() > 0) { work = (revisionWork - 20) / files.size(); for (Iterator fit = files.iterator(); fit.hasNext();) { File file = (File) fit.next(); monitor.subTask("Scanning java file...."); TreeSet commentSet = (TreeSet) fileCommentsMap.get(file.getAbsolutePath()); if (commentSet == null) { commentSet = new TreeSet(); fileCommentsMap.put(file.getAbsolutePath(), commentSet); } FileReader reader = new FileReader(file); StringBuilder builder = new StringBuilder(); char[] buffer = new char[1024]; int read = 0; while ((read = reader.read(buffer)) >= 0) { builder.append(buffer, 0, read); } reader.close(); ISVNAnnotations ann = null; try { //get blame information. List fileLogs = getLogEntries(file, client, repository); //don't do extra work if this file doesn't have a log for this revision. if (!checkRevision(fileLogs, revision)) { monitor.worked(work); //System.out.println("Skipped " + file.getAbsolutePath() + " revision " + revision.getNumber()); continue; } ann = client.annotate(file, revision, revision); } catch (SVNClientException e) { } catch (TeamException e) { } if (monitor.isCanceled()) { return Status.CANCEL_STATUS; } SubProgressMonitor scanMonitor = new SubProgressMonitor(revMonitor, work); Stats s = SimpleJavaCodeScanner.scan(builder.toString(), scanMonitor); if (monitor.isCanceled()) { return Status.CANCEL_STATUS; } monitor.worked(work); out.println("New/Changed Tags:"); for (int ci = 0; ci < s.TAGS.length; ci++) { Comment c = s.TAGS[ci]; if (!commentSet.contains(c)) { commentSet.add(c); String author = getAuthor(c, ann); out.println(c.toString() + "\tauthor=" + author); } } out.println("New/Changed Tasks:"); for (int ci = 0; ci < s.TASKS.length; ci++) { Comment c = s.TASKS[ci]; if (!commentSet.contains(c)) { commentSet.add(c); String author = getAuthor(c, ann); out.println(c.toString() + "\tauthor=" + author); } } out.println("New/Changed Other:"); for (int ci = 0; ci < s.NONTAGS.length; ci++) { Comment c = s.NONTAGS[ci]; if (!commentSet.contains(c)) { commentSet.add(c); String author = getAuthor(c, ann); out.println(c.toString() + "\tauthor=" + author); } } if (monitor.isCanceled()) { return Status.CANCEL_STATUS; } } } if (work == 0) { revMonitor.worked(revisionWork - 10); } monitor.subTask("Sending and Deleting temporary files..."); out.close(); sendFile(tempFile); deleteTemps(tempdir); if (monitor.isCanceled()) { return Status.CANCEL_STATUS; } revMonitor.done(); monitor.worked(revisionWork - 20); } } catch (SVNException e) { return new Status(IStatus.ERROR, SVNStatistics.PLUGIN_ID, 0, e.getMessage(), e); } catch (IOException e) { return new Status(IStatus.ERROR, SVNStatistics.PLUGIN_ID, 0, e.getMessage(), e); } } return Status.OK_STATUS; }
From source file:fr.certu.chouette.command.Command.java
/** * @param manager//from w ww . ja v a 2 s. c o m * @param parameters * @return */ private List<NeptuneIdentifiedObject> executeImport(INeptuneManager<NeptuneIdentifiedObject> manager, Map<String, List<String>> parameters) { String reportFileName = getSimpleString(parameters, "reportfile", ""); String reportFormat = getSimpleString(parameters, "reportformat", "txt"); boolean append = getBoolean(parameters, "append"); String format = getSimpleString(parameters, "format"); PrintStream stream = System.out; String encoding = Charset.defaultCharset().toString(); if (!reportFileName.isEmpty()) { try { if (reportFormat.equals("json")) { encoding = "UTF-8"; } stream = new PrintStream(new FileOutputStream(new File(reportFileName), append), true, encoding); } catch (IOException e) { System.err.println("cannot open file :" + reportFileName + " " + e.getMessage()); reportFileName = ""; } } try { List<FormatDescription> formats = manager.getImportFormats(null); FormatDescription description = null; for (FormatDescription formatDescription : formats) { if (formatDescription.getName().equalsIgnoreCase(format)) { description = formatDescription; break; } } if (description == null) { throw new IllegalArgumentException( "format " + format + " unavailable, check command getImportFormats for list "); } List<ParameterValue> values = new ArrayList<ParameterValue>(); for (ParameterDescription desc : description.getParameterDescriptions()) { String name = desc.getName(); String key = name.toLowerCase(); List<String> vals = parameters.get(key); if (vals == null) { if (desc.isMandatory()) { throw new IllegalArgumentException( "parameter -" + name + " is required, check command getImportFormats for list "); } } else { if (desc.isCollection()) { ListParameterValue val = new ListParameterValue(name); switch (desc.getType()) { case FILEPATH: val.setFilepathList(vals); break; case STRING: val.setStringList(vals); break; case FILENAME: val.setFilenameList(vals); break; } values.add(val); } else { if (vals.size() != 1) { throw new IllegalArgumentException("parameter -" + name + " must be unique, check command getImportFormats for list "); } String simpleval = vals.get(0); SimpleParameterValue val = new SimpleParameterValue(name); switch (desc.getType()) { case FILEPATH: val.setFilepathValue(simpleval); break; case STRING: val.setStringValue(simpleval); break; case FILENAME: val.setFilenameValue(simpleval); break; case BOOLEAN: val.setBooleanValue(Boolean.parseBoolean(simpleval)); break; case INTEGER: val.setIntegerValue(Long.parseLong(simpleval)); break; case DATE: val.setDateValue(toCalendar(simpleval)); break; } values.add(val); } } } ReportHolder holder = new ReportHolder(); List<NeptuneIdentifiedObject> beans = manager.doImport(null, format, values, holder); if (holder.getReport() != null) { Report r = holder.getReport(); if (reportFormat.equals("json")) { stream.println(r.toJSON()); } else { stream.println(r.getLocalizedMessage()); printItems(stream, "", r.getItems()); } } if (beans == null || beans.isEmpty()) { System.out.println("import failed"); } else { System.out.println("beans count = " + beans.size()); } return beans; } catch (ChouetteException e) { logger.error(e.getMessage()); Throwable caused = e.getCause(); while (caused != null) { logger.error("caused by " + caused.getMessage()); caused = caused.getCause(); } throw new RuntimeException("import failed , see log for details"); } finally { if (!reportFileName.isEmpty()) { stream.close(); } } }
From source file:edu.cmu.tetrad.search.TestIndTestConditionalCorrelation.java
public void test13() { try {//from ww w.j a v a2 s . c om // PrintStream out = new PrintStream("/Users/josephramsey/test11out.txt"); // PrintStream out = new PrintStream("/home/jdramsey/test11out.txt"); // PrintStream out = new PrintStream("/home/jdramsey/test10out.txt"); PrintStream out = System.out; String _dir = "/Users/josephramsey/Documents/LAB_NOTEBOOK.2012.04.20/2013.11.23/roidata_for_joe/"; File dir = new File(_dir); File[] files = dir.listFiles(); DataSet dataSet = null; for (int i = 0; i < 20; i++) {//files.length; i++) { File file = files[i]; File file2 = new File(file, "roidata"); File[] files2 = file2.listFiles(); File file3 = files2[0]; DataReader reader = new DataReader(); reader.setVariablesSupplied(false); reader.setDelimiter(DelimiterType.WHITESPACE); DataSet _dataSet = reader.parseTabular(file3); if (dataSet == null) { dataSet = _dataSet; } else { dataSet = DataUtils.concatenateData(dataSet, _dataSet); } } { // Pc pc = new Pc(new IndTestFisherZ(dataSet, 0.001)); PcStable pc = new PcStable(new IndTestFisherZ(dataSet, 0.001)); // Cpc pc = new Cpc(new IndTestFisherZ(dataSet, 0.001)); // PcStable pc = new PcStable(new IndTestConditionalCorrelation(dataSet, .001)); // pc.setDepth(3); Graph graph = pc.search(); System.out.println(graph); Lofs2 lofs = new Lofs2(graph, Collections.singletonList(dataSet)); lofs.setRule(Lofs2.Rule.R3); Graph graph2 = lofs.orient(); // lofs.setRule(Lofs2.Rule.RSkew); // graph = lofs.orie2nt(); System.out.println("With R3" + graph2); graph2 = GraphUtils.replaceNodes(graph2, graph.getNodes()); int countSame = 0; int countBothDirected = 0; for (Edge edge1 : graph.getEdges()) { Node n1 = edge1.getNode1(); Node n2 = edge1.getNode2(); Edge edge2 = graph2.getEdge(n1, n2); if (Edges.isDirectedEdge(edge1) && Edges.isDirectedEdge(edge2)) { countBothDirected++; if (edge1.equals(edge2)) { countSame++; } } } System.out.println(countSame + " of " + countBothDirected); } out.close(); } catch (Exception e) { e.printStackTrace(); } }
From source file:org.alfresco.repo.tenant.TenantInterpreter.java
/** * Execute a single command using the BufferedReader passed in for any data needed. * * TODO: Use decent parser!/*from www . j a va 2 s .co m*/ * * @param line The unparsed command * @return The textual output of the command. */ public String executeCommand(String line) throws IOException { String[] command = line.split(" "); if (command.length == 0) { command = new String[1]; command[0] = line; } ByteArrayOutputStream bout = new ByteArrayOutputStream(); PrintStream out = new PrintStream(bout); // repeat last command? if (command[0].equals("r")) { if (lastCommand == null) { return "No command entered yet."; } return "repeating command " + lastCommand + "\n\n" + executeCommand(lastCommand); } // remember last command lastCommand = line; // execute command if (command[0].equals("help")) { String helpFile = I18NUtil.getMessage("tenant_console.help"); ClassPathResource helpResource = new ClassPathResource(helpFile); byte[] helpBytes = new byte[500]; InputStream helpStream = helpResource.getInputStream(); try { int read = helpStream.read(helpBytes); while (read != -1) { bout.write(helpBytes, 0, read); read = helpStream.read(helpBytes); } } finally { helpStream.close(); } } else if (command[0].equals("show")) { if (command.length < 2) { return "Syntax Error, try 'help'.\n"; } else if (command[1].equals("tenants")) { List<Tenant> tenants = tenantAdminService.getAllTenants(); for (Tenant tenant : tenants) { if (tenant.isEnabled()) { String contentRoot = tenant.getRootContentStoreDir(); out.println("Enabled - Tenant: " + tenant.getTenantDomain() + " (" + contentRoot + ")"); } } out.println(""); for (Tenant tenant : tenants) { if (!tenant.isEnabled()) { String contentRoot = tenant.getRootContentStoreDir(); out.println("Disabled - Tenant: " + tenant.getTenantDomain() + " (" + contentRoot + ")"); } } } else if (command[1].equals("tenant")) { if (command.length != 3) { return "Syntax Error, try 'help'.\n"; } String tenantDomain = new String(command[2]).toLowerCase(); Tenant tenant = tenantAdminService.getTenant(tenantDomain); String contentRoot = tenant.getRootContentStoreDir(); if (tenant.isEnabled()) { out.println("Enabled - Tenant: " + tenant.getTenantDomain() + " (" + contentRoot + ")"); } else { out.println("Disabled - Tenant: " + tenant.getTenantDomain() + " (" + contentRoot + ")"); } } else { return "No such sub-command, try 'help'.\n"; } } else if (command[0].equals("create")) { if ((command.length < 3) || (command.length > 5)) { return "Syntax Error, try 'help'.\n"; } String newTenant = new String(command[1]).toLowerCase(); char[] tenantAdminRawPassword = new String(command[2]).toCharArray(); String contentRoot = null; if (command.length >= 4) { contentRoot = new String(command[3]); if ("null".equals(contentRoot)) { contentRoot = null; } } String dbUrl = null; if (command.length >= 5) { // experimental (unsupported) dbUrl = new String(command[4]); if ("null".equals(dbUrl)) { dbUrl = null; } } tenantAdminService.createTenant(newTenant, tenantAdminRawPassword, contentRoot, dbUrl); out.println("created tenant: " + newTenant); } else if (command[0].equals("import")) { if ((command.length != 3) && (command.length != 4)) { return "Syntax Error, try 'help'.\n"; } String newTenant = new String(command[1]).toLowerCase(); File directorySource = new File(command[2]); String contentRoot = null; if (command.length == 4) { contentRoot = new String(command[3]); } tenantAdminService.importTenant(newTenant, directorySource, contentRoot); out.println("imported tenant: " + newTenant); } else if (command[0].equals("export")) { if (command.length != 3) { return "Syntax Error, try 'help'.\n"; } String tenant = new String(command[1]).toLowerCase(); File directoryDestination = new File(command[2]); tenantAdminService.exportTenant(tenant, directoryDestination); out.println("exported tenant: " + tenant); } // TODO - not fully working yet else if (command[0].equals("delete")) { if (command.length != 2) { return "Syntax Error, try 'help'.\n"; } String tenantDomain = new String(command[1]).toLowerCase(); tenantAdminService.deleteTenant(tenantDomain); out.println("Deleted tenant: " + tenantDomain); } else if (command[0].equals("enable")) { if (command.length != 2) { return "Syntax Error, try 'help'.\n"; } String tenantDomain = new String(command[1]).toLowerCase(); tenantAdminService.enableTenant(tenantDomain); out.println("Enabled tenant: " + tenantDomain); } else if (command[0].equals("disable")) { if (command.length != 2) { return "Syntax Error, try 'help'.\n"; } String tenantDomain = new String(command[1]).toLowerCase(); tenantAdminService.disableTenant(tenantDomain); out.println("Disabled tenant: " + tenantDomain); } else if (command[0].equals("changeAdminPassword")) { if (command.length != 3) { return "Syntax Error, try 'help'.\n"; } String tenantDomain = new String(command[1]).toLowerCase(); final String newPassword = new String(command[2]); final String tenantAdminUsername = tenantService.getDomainUser(getBaseAdminUsername(), tenantDomain); AuthenticationUtil.runAs(new RunAsWork<Object>() { public Object doWork() throws Exception { authenticationService.setAuthentication(tenantAdminUsername, newPassword.toCharArray()); return null; } }, tenantAdminUsername); } else { return "No such command, try 'help'.\n"; } out.flush(); String retVal = new String(bout.toByteArray()); out.close(); return retVal; }
From source file:edu.cmu.tetrad.search.TestIndTestConditionalCorrelation.java
public void test10() { try {// w ww .ja va 2s . com // PrintStream out = new PrintStream("/Users/josephramsey/test10out.txt"); // PrintStream out = new PrintStream("/home/jdramsey/test10out.txt"); PrintStream out = System.out; int numVariables = 20; int numRuns = 20; NumberFormat nf = new DecimalFormat("0.00"); for (int modelIndex = 1; modelIndex <= 14; modelIndex++) { double sumAP1 = 0.0; double sumAR1 = 0.0; double sumAP2 = 0.0; double sumAR2 = 0.0; int sumAP1N = 0; int sumAR1N = 0; int sumAP2N = 0; int sumAR2N = 0; for (int r = 0; r < numRuns; r++) { GeneralizedSemIm im = makeTestIm4(numVariables, modelIndex); out.println(im); SemGraph gTrue = im.getGeneralizedSemPm().getGraph(); gTrue.setShowErrorTerms(false); DataSet data = im.simulateData(1000, false); PcStable pc = new PcStable(new IndTestConditionalCorrelation(data, .05)); Graph graph = pc.search(); // Goes to report. out.println(graph); graph = GraphUtils.replaceNodes(graph, gTrue.getNodes()); int adjFn = adjacenciesComplement(gTrue, graph); int adjFp = adjacenciesComplement(graph, gTrue); int truePosAdj = truePositivesAdj(gTrue, graph); double adjPrecision = truePosAdj / (double) (truePosAdj + adjFp); double adjRecall = truePosAdj / (double) (truePosAdj + adjFn); if (!Double.isNaN(adjPrecision)) { sumAP1 += adjPrecision; sumAP1N++; } if (!Double.isNaN(adjRecall)) { sumAR1 += adjRecall; sumAR1N++; } out.println("Model # " + modelIndex + " AP (CCI) = " + adjPrecision); out.println("Model # " + modelIndex + " AR (CCI) = " + adjRecall); Pc pc2 = new Pc(new IndTestFisherZ(data, 0.01)); Graph graph2 = pc2.search(); // Should go to the report. out.println(graph2); graph2 = GraphUtils.replaceNodes(graph2, gTrue.getNodes()); int adjFn2 = adjacenciesComplement(gTrue, graph2); int adjFp2 = adjacenciesComplement(graph2, gTrue); int truePosAdj2 = truePositivesAdj(gTrue, graph2); double adjPrecision2 = truePosAdj2 / (double) (truePosAdj2 + adjFp2); double adjRecall2 = truePosAdj2 / (double) (truePosAdj2 + adjFn2); if (!Double.isNaN(adjPrecision2)) { sumAP2 += adjPrecision2; sumAP2N++; } if (!Double.isNaN(adjRecall2)) { sumAR2 += adjRecall2; sumAR2N++; } out.println("Model # " + modelIndex + " AP (Fisher Z) = " + adjPrecision2); out.println("Model # " + modelIndex + " AR (Fisher Z) = " + adjRecall2); } out.println("Model # " + modelIndex + " Average AP (CCI) = " + nf.format(sumAP1 / sumAP1N)); out.println("Model # " + modelIndex + " Average AR (CCI) = " + nf.format(sumAR1 / sumAR1N)); out.println("Model # " + modelIndex + " Average AP (Fisher Z) = " + nf.format(sumAP2 / sumAP2N)); out.println("Model # " + modelIndex + " Average AR (Fisher Z) = " + nf.format(sumAR2 / sumAR2N)); } out.close(); } catch (Exception e) { e.printStackTrace(); } }
From source file:fr.inrialpes.exmo.align.cli.ExtGroupEval.java
public void printHTML(Vector<Vector<Object>> result, PrintStream writer) { // variables for computing iterative harmonic means int expected = 0; // expected so far int foundVect[]; // found so far double symVect[]; // symmetric similarity double effVect[]; // effort-based similarity double precOrVect[]; // precision-oriented similarity double recOrVect[]; // recall-oriented similarity fsize = format.length();//w ww . j a v a 2 s. c o m try { Formatter formatter = new Formatter(writer); // Print the header writer.println("<html><head></head><body>"); writer.println("<table border='2' frame='sides' rules='groups'>"); writer.println("<colgroup align='center' />"); // for each algo <td spancol='2'>name</td> for (String m : listAlgo) { writer.println("<colgroup align='center' span='" + 2 * fsize + "' />"); } // For each file do a writer.println("<thead valign='top'><tr><th>algo</th>"); // for each algo <td spancol='2'>name</td> for (String m : listAlgo) { writer.println("<th colspan='" + ((2 * fsize)) + "'>" + m + "</th>"); } writer.println("</tr></thead><tbody><tr><td>test</td>"); // for each algo <td>Prec.</td><td>Rec.</td> for (String m : listAlgo) { for (int i = 0; i < fsize; i++) { if (format.charAt(i) == 's') { writer.println("<td colspan='2'><center>Symmetric</center></td>"); } else if (format.charAt(i) == 'e') { writer.println("<td colspan='2'><center>Effort</center></td>"); } else if (format.charAt(i) == 'p') { writer.println("<td colspan='2'><center>Prec. orient.</center></td>"); } else if (format.charAt(i) == 'r') { writer.println("<td colspan='2'><center>Rec. orient.</center></td>"); } } //writer.println("<td>Prec.</td><td>Rec.</td>"); } writer.println("</tr></tbody><tbody>"); foundVect = new int[size]; symVect = new double[size]; effVect = new double[size]; precOrVect = new double[size]; recOrVect = new double[size]; for (int k = size - 1; k >= 0; k--) { foundVect[k] = 0; symVect[k] = 0.; effVect[k] = 0.; precOrVect[k] = 0.; recOrVect[k] = 0.; } // </tr> // For each directory <tr> boolean colored = false; for (Vector<Object> test : result) { int nexpected = -1; if (colored == true && color != null) { colored = false; writer.println("<tr bgcolor=\"" + color + "\">"); } else { colored = true; writer.println("<tr>"); } ; // Print the directory <td>bla</td> writer.println("<td>" + (String) test.get(0) + "</td>"); // For each record print the values <td>bla</td> Enumeration<Object> f = test.elements(); f.nextElement(); for (int k = 0; f.hasMoreElements(); k++) { ExtPREvaluator eval = (ExtPREvaluator) f.nextElement(); if (eval != null) { // iterative H-means computation if (nexpected == -1) { nexpected = eval.getExpected(); expected += nexpected; } // If foundVect is -1 then results are invalid if (foundVect[k] != -1) foundVect[k] += eval.getFound(); for (int i = 0; i < fsize; i++) { writer.print("<td>"); if (format.charAt(i) == 's') { formatter.format("%1.2f", eval.getSymPrecision()); writer.print("</td><td>"); formatter.format("%1.2f", eval.getSymRecall()); symVect[k] += eval.getSymSimilarity(); } else if (format.charAt(i) == 'e') { formatter.format("%1.2f", eval.getEffPrecision()); writer.print("</td><td>"); formatter.format("%1.2f", eval.getEffRecall()); effVect[k] += eval.getEffSimilarity(); } else if (format.charAt(i) == 'p') { formatter.format("%1.2f", eval.getPrecisionOrientedPrecision()); writer.print("</td><td>"); formatter.format("%1.2f", eval.getPrecisionOrientedRecall()); precOrVect[k] += eval.getPrecisionOrientedSimilarity(); } else if (format.charAt(i) == 'r') { formatter.format("%1.2f", eval.getRecallOrientedPrecision()); writer.print("</td><td>"); formatter.format("%1.2f", eval.getRecallOrientedRecall()); recOrVect[k] += eval.getRecallOrientedSimilarity(); } writer.print("</td>"); } } else { for (int i = 0; i < fsize; i++) writer.print("<td>n/a</td>"); } } writer.println("</tr>"); } writer.print("<tr bgcolor=\"yellow\"><td>H-mean</td>"); int k = 0; for (String m : listAlgo) { if (foundVect[k] != -1) { for (int i = 0; i < fsize; i++) { writer.print("<td>"); if (format.charAt(i) == 's') { formatter.format("%1.2f", symVect[k] / foundVect[k]); writer.print("</td><td>"); formatter.format("%1.2f", symVect[k] / expected); } else if (format.charAt(i) == 'e') { formatter.format("%1.2f", effVect[k] / foundVect[k]); writer.print("</td><td>"); formatter.format("%1.2f", effVect[k] / expected); } else if (format.charAt(i) == 'p') { formatter.format("%1.2f", precOrVect[k] / foundVect[k]); writer.print("</td><td>"); formatter.format("%1.2f", precOrVect[k] / expected); } else if (format.charAt(i) == 'r') { formatter.format("%1.2f", recOrVect[k] / foundVect[k]); writer.print("</td><td>"); formatter.format("%1.2f", recOrVect[k] / expected); } writer.println("</td>"); } } else { writer.println("<td colspan='2'><center>Error</center></td>"); } //}; k++; } writer.println("</tr>"); writer.println("</tbody></table>"); writer.println("<p><small>n/a: result alignment not provided or not readable<br />"); writer.println("NaN: division per zero, likely due to empty alignent.</small></p>"); writer.println("</body></html>"); } catch (Exception ex) { logger.debug("IGNORED Exception", ex); } finally { writer.flush(); writer.close(); } }
From source file:it.unimi.di.big.mg4j.tool.Scan.java
/** * Runs in parallel a number of instances. * /*w w w. ja v a 2 s . c o m*/ * <p>This commodity method takes care of instantiating one instance per indexed field, and to * pass the right information to each instance. All options are common to all fields, except for * the number of occurrences in a batch, which can be tuned for each field separately. * * @param ioFactory the factory that will be used to perform I/O. * @param basename the index basename. * @param documentSequence a document sequence. * @param completeness the completeness level of this run. * @param termProcessor the term processor for this index. * @param builder if not {@code null}, a builder that will be used to create new collection built using <code>documentSequence</code>. * @param bufferSize the buffer size used in all I/O. * @param documentsPerBatch the number of documents that we should try to put in each segment. * @param maxTerms the maximum number of overall (i.e., cross-field) terms in a batch. * @param indexedField the fields that should be indexed, in increasing order. * @param virtualDocumentResolver the array of virtual document resolvers to be used, parallel * to <code>indexedField</code>: it can safely contain anything (even {@code null}) * in correspondence to non-virtual fields, and can safely be {@code null} if no fields * are virtual. * @param virtualGap the array of virtual field gaps to be used, parallel to * <code>indexedField</code>: it can safely contain anything in correspondence to non-virtual * fields, and can safely be {@code null} if no fields are virtual. * @param mapFile the name of a file containing a map to be applied to document indices. * @param logInterval the minimum time interval between activity logs in milliseconds. * @param tempDirName a directory for temporary files. * @throws IOException * @throws ConfigurationException */ @SuppressWarnings("unchecked") public static void run(final IOFactory ioFactory, final String basename, final DocumentSequence documentSequence, final Completeness completeness, final TermProcessor termProcessor, final DocumentCollectionBuilder builder, final int bufferSize, final int documentsPerBatch, final int maxTerms, final int[] indexedField, final VirtualDocumentResolver[] virtualDocumentResolver, final int[] virtualGap, final String mapFile, final long logInterval, final String tempDirName) throws ConfigurationException, IOException { final boolean building = builder != null; final int numberOfIndexedFields = indexedField.length; if (numberOfIndexedFields == 0) throw new IllegalArgumentException("You must specify at least one field"); final DocumentFactory factory = documentSequence.factory(); final File tempDir = tempDirName == null ? null : new File(tempDirName); for (int i = 0; i < indexedField.length; i++) if (factory.fieldType(indexedField[i]) == DocumentFactory.FieldType.VIRTUAL && (virtualDocumentResolver == null || virtualDocumentResolver[i] == null)) throw new IllegalArgumentException( "No resolver was associated with virtual field " + factory.fieldName(indexedField[i])); if (mapFile != null && ioFactory != IOFactory.FILESYSTEM_FACTORY) throw new IllegalStateException("Remapped indices currently do not support I/O factories"); final int[] map = mapFile != null ? BinIO.loadInts(mapFile) : null; final Scan[] scan = new Scan[numberOfIndexedFields]; // To scan textual content final PayloadAccumulator[] accumulator = new PayloadAccumulator[numberOfIndexedFields]; // To accumulate // document data final ProgressLogger pl = new ProgressLogger(LOGGER, logInterval, TimeUnit.MILLISECONDS, "documents"); if (documentSequence instanceof DocumentCollection) pl.expectedUpdates = ((DocumentCollection) documentSequence).size(); final PrintStream titleStream = new PrintStream(basename + TITLES_EXTENSION, "UTF-8"); for (int i = 0; i < numberOfIndexedFields; i++) { final String fieldName = factory.fieldName(indexedField[i]); switch (factory.fieldType(indexedField[i])) { case TEXT: scan[i] = new Scan(ioFactory, basename + '-' + fieldName, fieldName, completeness, termProcessor, map != null ? IndexingType.REMAPPED : IndexingType.STANDARD, 0, 0, bufferSize, builder, tempDir); break; case VIRTUAL: scan[i] = new Scan(ioFactory, basename + '-' + fieldName, fieldName, completeness, termProcessor, IndexingType.VIRTUAL, virtualDocumentResolver[i].numberOfDocuments(), virtualGap[i], bufferSize, builder, tempDir); break; case DATE: accumulator[i] = new PayloadAccumulator(ioFactory, basename + '-' + fieldName, new DatePayload(), fieldName, map != null ? IndexingType.REMAPPED : IndexingType.STANDARD, documentsPerBatch, tempDir); break; case INT: accumulator[i] = new PayloadAccumulator(ioFactory, basename + '-' + fieldName, new IntegerPayload(), fieldName, map != null ? IndexingType.REMAPPED : IndexingType.STANDARD, documentsPerBatch, tempDir); break; default: } } if (building) builder.open("@0"); // First batch pl.displayFreeMemory = true; pl.start("Indexing documents..."); DocumentIterator iterator = documentSequence.iterator(); Reader reader; WordReader wordReader; List<VirtualDocumentFragment> fragments; Document document; int documentPointer = 0, documentsInBatch = 0; long batchStartTime = System.currentTimeMillis(); boolean outOfMemoryError = false; final MutableString title = new MutableString(); while ((document = iterator.nextDocument()) != null) { long overallTerms = 0; if (document.title() != null) { title.replace(document.title()); title.replace(ScanMetadata.LINE_TERMINATORS, ScanMetadata.SPACES); titleStream.print(title); } titleStream.println(); if (building) builder.startDocument(document.title(), document.uri()); for (int i = 0; i < numberOfIndexedFields; i++) { switch (factory.fieldType(indexedField[i])) { case TEXT: reader = (Reader) document.content(indexedField[i]); wordReader = document.wordReader(indexedField[i]); wordReader.setReader(reader); if (building) builder.startTextField(); scan[i].processDocument(map != null ? map[documentPointer] : documentPointer, wordReader); if (building) builder.endTextField(); overallTerms += scan[i].numTerms; break; case VIRTUAL: fragments = (List<VirtualDocumentFragment>) document.content(indexedField[i]); wordReader = document.wordReader(indexedField[i]); virtualDocumentResolver[i].context(document); for (VirtualDocumentFragment fragment : fragments) { long virtualDocumentPointer = virtualDocumentResolver[i] .resolve(fragment.documentSpecifier()); if (virtualDocumentPointer < 0) continue; // ALERT: we must rewrite remapping to work with long-sized document pointers. if (map != null) virtualDocumentPointer = map[(int) virtualDocumentPointer]; wordReader.setReader(new FastBufferedReader(fragment.text())); scan[i].processDocument((int) virtualDocumentPointer, wordReader); } if (building) builder.virtualField(fragments); overallTerms += scan[i].numTerms; break; default: Object o = document.content(indexedField[i]); accumulator[i].processData(map != null ? map[documentPointer] : documentPointer, o); if (building) builder.nonTextField(o); break; } if (scan[i] != null && scan[i].outOfMemoryError) outOfMemoryError = true; } if (building) builder.endDocument(); documentPointer++; documentsInBatch++; document.close(); pl.update(); long percAvailableMemory = 100; boolean compacted = false; if ((documentPointer & 0xFF) == 0) { // We try compaction if we detect less than PERC_AVAILABLE_MEMORY_CHECK memory available percAvailableMemory = Util.percAvailableMemory(); if (!outOfMemoryError && percAvailableMemory < PERC_AVAILABLE_MEMORY_CHECK) { LOGGER.info("Starting compaction... (" + percAvailableMemory + "% available)"); compacted = true; Util.compactMemory(); percAvailableMemory = Util.percAvailableMemory(); LOGGER.info("Compaction completed (" + percAvailableMemory + "% available)"); } } if (outOfMemoryError || overallTerms >= maxTerms || documentsInBatch == documentsPerBatch || (compacted && percAvailableMemory < PERC_AVAILABLE_MEMORY_DUMP)) { if (outOfMemoryError) LOGGER.warn("OutOfMemoryError during buffer reallocation: writing a batch of " + documentsInBatch + " documents"); else if (overallTerms >= maxTerms) LOGGER.warn("Too many terms (" + overallTerms + "): writing a batch of " + documentsInBatch + " documents"); else if (compacted && percAvailableMemory < PERC_AVAILABLE_MEMORY_DUMP) LOGGER.warn("Available memory below " + PERC_AVAILABLE_MEMORY_DUMP + "%: writing a batch of " + documentsInBatch + " documents"); long occurrences = 0; for (int i = 0; i < numberOfIndexedFields; i++) { switch (factory.fieldType(indexedField[i])) { case TEXT: case VIRTUAL: occurrences += scan[i].dumpBatch(); scan[i].openSizeBitStream(); break; default: accumulator[i].writeData(); } } if (building) { builder.close(); builder.open("@" + scan[0].batch); } LOGGER.info("Last set of batches indexed at " + Util.format((1000. * occurrences) / (System.currentTimeMillis() - batchStartTime)) + " occurrences/s"); batchStartTime = System.currentTimeMillis(); documentsInBatch = 0; outOfMemoryError = false; } } iterator.close(); titleStream.close(); if (builder != null) builder.close(); for (int i = 0; i < numberOfIndexedFields; i++) { switch (factory.fieldType(indexedField[i])) { case TEXT: case VIRTUAL: scan[i].close(); break; default: accumulator[i].close(); break; } } documentSequence.close(); pl.done(); if (building) { final String name = new File(builder.basename()).getName(); final String[] collectionName = new String[scan[0].batch]; for (int i = scan[0].batch; i-- != 0;) collectionName[i] = name + "@" + i + DocumentCollection.DEFAULT_EXTENSION; IOFactories.storeObject(ioFactory, new ConcatenatedDocumentCollection(collectionName), builder.basename() + DocumentCollection.DEFAULT_EXTENSION); } if (map != null && documentPointer != map.length) LOGGER.warn("The document sequence contains " + documentPointer + " documents, but the map contains " + map.length + " integers"); }
From source file:de.tudarmstadt.ukp.experiments.argumentation.convincingness.sampling.Step6GraphTransitivityCleaner.java
public GraphCleaningResults processSingleFile(File file, File outputDir, String prefix, Boolean collectGeneratedArgumentPairs) throws Exception { GraphCleaningResults result = new GraphCleaningResults(); File outFileTable = new File(outputDir, prefix + file.getName() + "_table.csv"); File outFileInfo = new File(outputDir, prefix + file.getName() + "_info.txt"); PrintStream psTable = new PrintStream(new FileOutputStream(outFileTable)); PrintStream psInfo = new PrintStream(new FileOutputStream(outFileInfo)); // load one topic/side List<AnnotatedArgumentPair> pairs = new ArrayList<>( (List<AnnotatedArgumentPair>) XStreamTools.getXStream().fromXML(file)); int fullDataSize = pairs.size(); // filter out missing gold data Iterator<AnnotatedArgumentPair> iterator = pairs.iterator(); while (iterator.hasNext()) { AnnotatedArgumentPair pair = iterator.next(); if (pair.getGoldLabel() == null) { iterator.remove();// ww w. ja v a2 s . c o m } // or we want to completely remove equal edges in advance! else if (this.removeEqualEdgesParam && "equal".equals(pair.getGoldLabel())) { iterator.remove(); } } // sort pairs by their weight this.argumentPairListSorter.sortArgumentPairs(pairs); int preFilteredDataSize = pairs.size(); // compute correlation between score threshold and number of removed edges double[] correlationEdgeWeights = new double[pairs.size()]; double[] correlationRemovedEdges = new double[pairs.size()]; // only cycles of length 0 to 5 are interesting (5+ are too big) Range<Integer> range = Range.between(0, 5); psTable.print( "EdgeWeightThreshold\tPairs\tignoredEdgesCount\tIsDAG\tTransitivityScoreMean\tTransitivityScoreMax\tTransitivityScoreSamples\tEdges\tNodes\t"); for (int j = range.getMinimum(); j <= range.getMaximum(); j++) { psTable.print("Cycles_" + j + "\t"); } psTable.println(); // store the indices of all pairs (edges) that have been successfully added without // generating cycles TreeSet<Integer> addedPairsIndices = new TreeSet<>(); // number of edges ignored as they generated cycles int ignoredEdgesCount = 0; Graph lastGraph = null; // flag that the first cycle was already processed boolean firstCycleAlreadyHit = false; for (int i = 1; i < pairs.size(); i++) { // now filter the finalArgumentPairList and add only pairs that have not generated cycles List<AnnotatedArgumentPair> subList = new ArrayList<>(); for (Integer index : addedPairsIndices) { subList.add(pairs.get(index)); } // and add the current at the end subList.add(pairs.get(i)); // what is the current lowest value of a pair weight? double weakestEdgeWeight = computeEdgeWeight(subList.get(subList.size() - 1), LAMBDA_PENALTY); // Graph graph = buildGraphFromArgumentPairs(finalArgumentPairList); int numberOfLoops; // map for storing cycles by their length TreeMap<Integer, TreeSet<String>> lengthCyclesMap = new TreeMap<>(); Graph graph = buildGraphFromArgumentPairs(subList); lastGraph = graph; List<List<Object>> cyclesInGraph = findCyclesInGraph(graph); DescriptiveStatistics transitivityScore = new DescriptiveStatistics(); if (cyclesInGraph.isEmpty()) { // we have DAG transitivityScore = computeTransitivityScores(graph); // update results result.maxTransitivityScore = (int) transitivityScore.getMax(); result.avgTransitivityScore = transitivityScore.getMean(); } numberOfLoops = cyclesInGraph.size(); // initialize map for (int r = range.getMinimum(); r <= range.getMaximum(); r++) { lengthCyclesMap.put(r, new TreeSet<String>()); } // we hit a loop if (numberOfLoops > 0) { // let's update the result if (!firstCycleAlreadyHit) { result.graphSizeEdgesBeforeFirstCycle = graph.getEdgeCount(); result.graphSizeNodesBeforeFirstCycle = graph.getNodeCount(); // find the shortest cycle int shortestCycleLength = Integer.MAX_VALUE; for (List<Object> cycle : cyclesInGraph) { shortestCycleLength = Math.min(shortestCycleLength, cycle.size()); } result.lengthOfFirstCircle = shortestCycleLength; result.pairsBeforeFirstCycle = i; firstCycleAlreadyHit = true; } // ignore this edge further ignoredEdgesCount++; // update counts of different cycles lengths for (List<Object> cycle : cyclesInGraph) { int currentSize = cycle.size(); // convert to sorted set of nodes List<String> cycleAsSortedIDs = new ArrayList<>(); for (Object o : cycle) { cycleAsSortedIDs.add(o.toString()); } Collections.sort(cycleAsSortedIDs); if (range.contains(currentSize)) { lengthCyclesMap.get(currentSize).add(cycleAsSortedIDs.toString()); } } } else { addedPairsIndices.add(i); } // we hit the first cycle // collect loop sizes StringBuilder loopsAsString = new StringBuilder(); for (int j = range.getMinimum(); j <= range.getMaximum(); j++) { // loopsAsString.append(j).append(":"); loopsAsString.append(lengthCyclesMap.get(j).size()); loopsAsString.append("\t"); } psTable.printf(Locale.ENGLISH, "%.4f\t%d\t%d\t%b\t%.2f\t%d\t%d\t%d\t%d\t%s%n", weakestEdgeWeight, i, ignoredEdgesCount, numberOfLoops == 0, Double.isNaN(transitivityScore.getMean()) ? 0d : transitivityScore.getMean(), (int) transitivityScore.getMax(), transitivityScore.getN(), graph.getEdgeCount(), graph.getNodeCount(), loopsAsString.toString().trim()); // update result result.finalGraphSizeEdges = graph.getEdgeCount(); result.finalGraphSizeNodes = graph.getNodeCount(); result.ignoredEdgesThatBrokeDAG = ignoredEdgesCount; // update stats for correlation correlationEdgeWeights[i] = weakestEdgeWeight; // correlationRemovedEdges[i] = (double) ignoredEdgesCount; // let's try: if we keep = 0, if we remove = 1 correlationRemovedEdges[i] = numberOfLoops == 0 ? 0.0 : 1.0; } psInfo.println("Original: " + fullDataSize + ", removed by MACE: " + (fullDataSize - preFilteredDataSize) + ", final: " + (preFilteredDataSize - ignoredEdgesCount) + " (removed: " + ignoredEdgesCount + ")"); double[][] matrix = new double[correlationEdgeWeights.length][]; for (int i = 0; i < correlationEdgeWeights.length; i++) { matrix[i] = new double[2]; matrix[i][0] = correlationEdgeWeights[i]; matrix[i][1] = correlationRemovedEdges[i]; } PearsonsCorrelation pearsonsCorrelation = new PearsonsCorrelation(matrix); double pValue = pearsonsCorrelation.getCorrelationPValues().getEntry(0, 1); double correlation = pearsonsCorrelation.getCorrelationMatrix().getEntry(0, 1); psInfo.printf(Locale.ENGLISH, "Correlation: %.3f, p-Value: %.4f%n", correlation, pValue); if (lastGraph == null) { throw new IllegalStateException("Graph is null"); } // close psInfo.close(); psTable.close(); // save filtered final gold data List<AnnotatedArgumentPair> finalArgumentPairList = new ArrayList<>(); for (Integer index : addedPairsIndices) { finalArgumentPairList.add(pairs.get(index)); } XStreamTools.toXML(finalArgumentPairList, new File(outputDir, prefix + file.getName())); // TODO: here, we can add newly generated edges from graph transitivity if (collectGeneratedArgumentPairs) { Set<GeneratedArgumentPair> generatedArgumentPairs = new HashSet<>(); // collect all arguments Map<String, Argument> allArguments = new HashMap<>(); for (ArgumentPair argumentPair : pairs) { allArguments.put(argumentPair.getArg1().getId(), argumentPair.getArg1()); allArguments.put(argumentPair.getArg2().getId(), argumentPair.getArg2()); } Graph finalGraph = buildGraphFromArgumentPairs(finalArgumentPairList); for (Edge e : finalGraph.getEdgeSet()) { e.setAttribute(WEIGHT, 1.0); } for (Node j : finalGraph) { for (Node k : finalGraph) { if (j != k) { // is there a path between? BellmanFord bfShortest = new BellmanFord(WEIGHT, j.getId()); bfShortest.init(finalGraph); bfShortest.compute(); Path shortestPath = bfShortest.getShortestPath(k); if (shortestPath.size() > 0) { // we have a path GeneratedArgumentPair ap = new GeneratedArgumentPair(); Argument arg1 = allArguments.get(j.getId()); if (arg1 == null) { throw new IllegalStateException("Cannot find argument " + j.getId()); } ap.setArg1(arg1); Argument arg2 = allArguments.get(k.getId()); if (arg2 == null) { throw new IllegalStateException("Cannot find argument " + k.getId()); } ap.setArg2(arg2); ap.setGoldLabel("a1"); generatedArgumentPairs.add(ap); } } } } // and now add the reverse ones Set<GeneratedArgumentPair> generatedReversePairs = new HashSet<>(); for (GeneratedArgumentPair pair : generatedArgumentPairs) { GeneratedArgumentPair ap = new GeneratedArgumentPair(); ap.setArg1(pair.getArg2()); ap.setArg2(pair.getArg1()); ap.setGoldLabel("a2"); generatedReversePairs.add(ap); } generatedArgumentPairs.addAll(generatedReversePairs); // and save it XStreamTools.toXML(generatedArgumentPairs, new File(outputDir, "generated_" + prefix + file.getName())); } result.fullPairsSize = fullDataSize; result.removedApriori = (fullDataSize - preFilteredDataSize); result.finalPairsRetained = finalArgumentPairList.size(); // save the final graph Graph outGraph = cleanCopyGraph(lastGraph); FileSinkDGS dgs1 = new FileSinkDGS(); File outFile = new File(outputDir, prefix + file.getName() + ".dgs"); System.out.println("Saved to " + outFile); FileWriter w1 = new FileWriter(outFile); dgs1.writeAll(outGraph, w1); w1.close(); return result; }