List of usage examples for weka.core.converters CSVSaver setFile
@Override public void setFile(File outputFile) throws IOException
From source file:cn.ict.zyq.bestConf.util.DataIOFile.java
License:Open Source License
/** * Save @param data to the CSV file at @param path *//*w ww. ja va2s. co m*/ public static void saveDataToCsvFile(String path, Instances data) throws IOException { System.out.println("\nSaving to file " + path + "..."); CSVSaver saver = new CSVSaver(); saver.setInstances(data); saver.setFile(new File(path)); saver.writeBatch(); }
From source file:com.sliit.neuralnetwork.RecurrentNN.java
public String trainModel(String modelName, String filePath, int outputs, int inputsTot) throws NeuralException { System.out.println("calling trainModel"); try {//from ww w. j a va 2s . c o m System.out.println("Neural Network Training start"); loadSaveNN(modelName, false); if (model == null) { buildModel(); } File fileGeneral = new File(filePath); CSVLoader loader = new CSVLoader(); loader.setSource(fileGeneral); Instances instances = loader.getDataSet(); instances.setClassIndex(instances.numAttributes() - 1); StratifiedRemoveFolds stratified = new StratifiedRemoveFolds(); String[] options = new String[6]; options[0] = "-N"; options[1] = Integer.toString(5); options[2] = "-F"; options[3] = Integer.toString(1); options[4] = "-S"; options[5] = Integer.toString(1); stratified.setOptions(options); stratified.setInputFormat(instances); stratified.setInvertSelection(false); Instances testInstances = Filter.useFilter(instances, stratified); stratified.setInvertSelection(true); Instances trainInstances = Filter.useFilter(instances, stratified); String directory = fileGeneral.getParent(); CSVSaver saver = new CSVSaver(); File trainFile = new File(directory + "/" + "normtrainadded.csv"); File testFile = new File(directory + "/" + "normtestadded.csv"); if (trainFile.exists()) { trainFile.delete(); } trainFile.createNewFile(); if (testFile.exists()) { testFile.delete(); } testFile.createNewFile(); saver.setFile(trainFile); saver.setInstances(trainInstances); saver.writeBatch(); saver = new CSVSaver(); saver.setFile(testFile); saver.setInstances(testInstances); saver.writeBatch(); SequenceRecordReader recordReader = new CSVSequenceRecordReader(0, ","); recordReader.initialize(new org.datavec.api.split.FileSplit(trainFile)); SequenceRecordReader testReader = new CSVSequenceRecordReader(0, ","); testReader.initialize(new org.datavec.api.split.FileSplit(testFile)); DataSetIterator iterator = new org.deeplearning4j.datasets.datavec.SequenceRecordReaderDataSetIterator( recordReader, 2, outputs, inputsTot, false); DataSetIterator testIterator = new org.deeplearning4j.datasets.datavec.SequenceRecordReaderDataSetIterator( testReader, 2, outputs, inputsTot, false); roc = new ArrayList<Map<String, Double>>(); String statMsg = ""; Evaluation evaluation; for (int i = 0; i < 100; i++) { if (i % 2 == 0) { model.fit(iterator); evaluation = model.evaluate(testIterator); } else { model.fit(testIterator); evaluation = model.evaluate(iterator); } Map<String, Double> map = new HashMap<String, Double>(); Map<Integer, Integer> falsePositives = evaluation.falsePositives(); Map<Integer, Integer> trueNegatives = evaluation.trueNegatives(); Map<Integer, Integer> truePositives = evaluation.truePositives(); Map<Integer, Integer> falseNegatives = evaluation.falseNegatives(); double fpr = falsePositives.get(1) / (falsePositives.get(1) + trueNegatives.get(1)); double tpr = truePositives.get(1) / (truePositives.get(1) + falseNegatives.get(1)); map.put("FPR", fpr); map.put("TPR", tpr); roc.add(map); statMsg = evaluation.stats(); iterator.reset(); testIterator.reset(); } loadSaveNN(modelName, true); System.out.println("ROC " + roc); return statMsg; } catch (Exception e) { e.printStackTrace(); System.out.println("Error ocuured while building neural netowrk :" + e.getMessage()); throw new NeuralException(e.getLocalizedMessage(), e); } }
From source file:com.sliit.normalize.NormalizeDataset.java
public String normalizeDataset() { System.out.println("start normalizing data"); String filePathOut = ""; try {// w ww.j a v a2 s. co m CSVLoader loader = new CSVLoader(); if (reducedDiemensionFile != null) { loader.setSource(reducedDiemensionFile); } else { if (tempFIle != null && tempFIle.exists()) { loader.setSource(tempFIle); } else { loader.setSource(csvFile); } } Instances dataInstance = loader.getDataSet(); Normalize normalize = new Normalize(); dataInstance.setClassIndex(dataInstance.numAttributes() - 1); normalize.setInputFormat(dataInstance); String directory = csvFile.getParent(); outputFile = new File(directory + "/" + "normalized" + csvFile.getName()); if (!outputFile.exists()) { outputFile.createNewFile(); } CSVSaver saver = new CSVSaver(); saver.setFile(outputFile); for (int i = 1; i < dataInstance.numInstances(); i++) { normalize.input(dataInstance.instance(i)); } normalize.batchFinished(); Instances outPut = new Instances(dataInstance, 0); for (int i = 1; i < dataInstance.numInstances(); i++) { outPut.add(normalize.output()); } Attribute attribute = dataInstance.attribute(outPut.numAttributes() - 1); for (int j = 0; j < attribute.numValues(); j++) { if (attribute.value(j).equals("normal.")) { outPut.renameAttributeValue(attribute, attribute.value(j), "0"); } else { outPut.renameAttributeValue(attribute, attribute.value(j), "1"); } } saver.setInstances(outPut); saver.writeBatch(); writeToNewFile(directory); filePathOut = directory + "norm" + csvFile.getName(); if (tempFIle != null) { tempFIle.delete(); } if (reducedDiemensionFile != null) { reducedDiemensionFile.delete(); } outputFile.delete(); } catch (IOException e) { log.error("Error occurred:" + e.getMessage()); } catch (Exception e) { log.error("Error occurred:" + e.getMessage()); } return filePathOut; }
From source file:com.sliit.normalize.NormalizeDataset.java
public boolean updateStringValues(Map<Integer, String> values) { System.out.println("updating String Values"); boolean status = false; try {/* www .j ava2s . c om*/ csv.setSource(csvFile); Instances dataInstance = csv.getDataSet(); for (int i = 0; i < dataInstance.numInstances(); i++) { if (values.containsKey(i)) { Attribute attribute = dataInstance.attribute(i); for (int j = 0; j < attribute.numValues(); j++) { dataInstance.renameAttributeValue(attribute, attribute.value(j), j + ""); } } } tempFIle = new File(csvFile.getParent() + "/temp.csv"); CSVSaver saver = new CSVSaver(); saver.setInstances(dataInstance); saver.setFile(tempFIle); saver.writeBatch(); } catch (IOException e) { log.error("Error occurred:" + e.getMessage()); } return status; }
From source file:es.bsc.autonomic.powermodeller.tools.classifiers.WekaWrapper.java
License:Apache License
public static DataSet processDataSet(DataSet ds, VariableParser parser) { String independent = ds.getIndependent(); if (independent == null) throw new WekaWrapperException("Independent variable is not set in dataset."); HashMap<String, String> expression_list = parser.getNewMetrics(); Instances data = convertDataSetToInstances(ds); try {//from w w w . ja v a 2s . c om // Apply filters for all the new variables for (Map.Entry<String, String> entry : expression_list.entrySet()) { String key = entry.getKey(); String value = entry.getValue(); logger.debug("Generating new variable " + key + " as " + value); AddExpression add_filter = new AddExpression(); add_filter.setName(key); add_filter.setExpression(value); add_filter.setInputFormat(data); data = useFilter(data, add_filter); } } catch (Exception e) { logger.error("Error while processing new variables", e); throw new WekaWrapperException("Error while processing new variables"); } // Iterate over all the columns and keep only the ones contained in variables list List<String> variables = parser.getColumns(); // Append independent variable to the list of variables to keep variables.add(independent); // Remove unneeded attributes try { // it's important to iterate from last to first, because when we remove // an instance, the rest shifts by one position. for (int i = data.numAttributes() - 1; i >= 0; i--) { String n = data.attribute(i).name(); if (!variables.contains(data.attribute(i).name())) { logger.trace("Deleting unnecessary attribute " + data.attribute(i).name()); data.deleteAttributeAt(i); } } data.toString(); } catch (Exception e) { logger.error("Error while removing unneeded variables", e); throw new WekaWrapperException("Error while removing unneeded variables"); } // Convert Instances in csv and return the new DataSet String new_path = CoreConfiguration.getNewCSVFileName(); try { CSVSaver saver = new CSVSaver(); saver.setInstances(data); saver.setFile(new File(new_path)); saver.writeBatch(); } catch (Exception e) { logger.error("Error while removing unneeded variables", e); throw new WekaWrapperException("Error while removing unneeded variables"); } DataSet ret = new DataSet(new_path); ret.setIndependent(independent); return ret; }
From source file:es.upm.dit.gsi.barmas.dataset.utils.DatasetSplitter.java
License:Open Source License
/** * @param folds//from w w w . ja va 2 s . com * @param minAgents * @param maxAgents * @param originalDatasetPath * @param outputDir * @param scenario * @param logger */ public void splitDataset(int folds, int minAgents, int maxAgents, String originalDatasetPath, String outputDir, String scenario, Logger logger) { int ratioint = (int) ((1 / (double) folds) * 100); double roundedratio = ((double) ratioint) / 100; // Look for essentials List<String[]> essentials = this.getEssentials(originalDatasetPath, logger); for (int fold = 0; fold < folds; fold++) { String outputDirWithRatio = outputDir + "/" + roundedratio + "testRatio/iteration-" + fold; File dir = new File(outputDirWithRatio); if (!dir.exists() || !dir.isDirectory()) { dir.mkdirs(); } logger.finer("--> splitDataset()"); logger.fine("Creating experiment.info..."); try { Instances originalData = this.getDataFromCSV(originalDatasetPath); originalData.randomize(new Random()); originalData.stratify(folds); // TestDataSet Instances testData = originalData.testCV(folds, fold); CSVSaver saver = new CSVSaver(); ArffSaver arffsaver = new ArffSaver(); File file = new File(outputDirWithRatio + File.separator + "test-dataset.csv"); if (!file.exists()) { saver.resetOptions(); saver.setInstances(testData); saver.setFile(file); saver.writeBatch(); } file = new File(outputDirWithRatio + File.separator + "test-dataset.arff"); if (!file.exists()) { arffsaver.resetOptions(); arffsaver.setInstances(testData); arffsaver.setFile(file); arffsaver.writeBatch(); } // BayesCentralDataset Instances trainData = originalData.trainCV(folds, fold); file = new File(outputDirWithRatio + File.separator + "bayes-central-dataset.csv"); if (!file.exists()) { saver.resetOptions(); saver.setInstances(trainData); saver.setFile(file); saver.writeBatch(); this.copyFileUsingApacheCommonsIO(file, new File( outputDirWithRatio + File.separator + "bayes-central-dataset-noEssentials.csv"), logger); CsvWriter w = new CsvWriter(new FileWriter(file, true), ','); for (String[] essential : essentials) { w.writeRecord(essential); } w.close(); } file = new File(outputDirWithRatio + File.separator + "bayes-central-dataset.arff"); if (!file.exists()) { arffsaver.resetOptions(); arffsaver.setInstances(trainData); arffsaver.setFile(file); arffsaver.writeBatch(); this.copyFileUsingApacheCommonsIO(file, new File( outputDirWithRatio + File.separator + "bayes-central-dataset-noEssentials.arff"), logger); CsvWriter w = new CsvWriter(new FileWriter(file, true), ','); for (String[] essential : essentials) { w.writeRecord(essential); } w.close(); } // Agent datasets CsvReader csvreader = new CsvReader(new FileReader(new File(originalDatasetPath))); csvreader.readHeaders(); String[] headers = csvreader.getHeaders(); csvreader.close(); for (int agents = minAgents; agents <= maxAgents; agents++) { this.createExperimentInfoFile(folds, agents, originalDatasetPath, outputDirWithRatio, scenario, logger); HashMap<String, CsvWriter> writers = new HashMap<String, CsvWriter>(); String agentsDatasetsDir = outputDirWithRatio + File.separator + agents + "agents"; HashMap<String, CsvWriter> arffWriters = new HashMap<String, CsvWriter>(); File f = new File(agentsDatasetsDir); if (!f.isDirectory()) { f.mkdirs(); } Instances copy = new Instances(trainData); copy.delete(); for (int i = 0; i < agents; i++) { String fileName = agentsDatasetsDir + File.separator + "agent-" + i + "-dataset.csv"; file = new File(fileName); if (!file.exists()) { CsvWriter writer = new CsvWriter(new FileWriter(fileName), ','); writer.writeRecord(headers); writers.put("AGENT" + i, writer); } fileName = agentsDatasetsDir + File.separator + "agent-" + i + "-dataset.arff"; file = new File(fileName); if (!file.exists()) { arffsaver.resetOptions(); arffsaver.setInstances(copy); arffsaver.setFile(new File(fileName)); arffsaver.writeBatch(); CsvWriter arffwriter = new CsvWriter(new FileWriter(fileName, true), ','); arffWriters.put("AGENT" + i, arffwriter); } logger.fine("AGENT" + i + " dataset created in csv and arff formats."); } // Append essentials to all for (String[] essential : essentials) { for (CsvWriter wr : writers.values()) { wr.writeRecord(essential); } for (CsvWriter arffwr : arffWriters.values()) { arffwr.writeRecord(essential); } } int agentCounter = 0; for (int j = 0; j < trainData.numInstances(); j++) { Instance instance = trainData.instance(j); CsvWriter writer = writers.get("AGENT" + agentCounter); CsvWriter arffwriter = arffWriters.get("AGENT" + agentCounter); String[] row = new String[instance.numAttributes()]; for (int a = 0; a < instance.numAttributes(); a++) { row[a] = instance.stringValue(a); } if (writer != null) { writer.writeRecord(row); } if (arffwriter != null) { arffwriter.writeRecord(row); } agentCounter++; if (agentCounter == agents) { agentCounter = 0; } } for (CsvWriter wr : writers.values()) { wr.close(); } for (CsvWriter arffwr : arffWriters.values()) { arffwr.close(); } } } catch (Exception e) { logger.severe("Exception while splitting dataset. ->"); logger.severe(e.getMessage()); System.exit(1); } logger.finest("Dataset for fold " + fold + " created."); } logger.finer("<-- splitDataset()"); }
From source file:etc.aloe.Aloe.java
License:Open Source License
protected void saveInstances(Instances instances, File outputFile) { try {// www .j av a 2s . c o m CSVSaver saver = new CSVSaver(); saver.setFile(outputFile); saver.setInstances(instances); saver.writeBatch(); System.out.println("Saved instances to " + outputFile); } catch (IOException e) { System.err.println("Error writing instances to " + outputFile); System.err.println("\t" + e.getMessage()); } }
From source file:linqs.gaia.model.oc.ncc.WekaClassifier.java
License:Open Source License
/** * Save instances using the weka CSV format * //from ww w .j a v a 2 s.c om * @param file CSV filename to write to. If specified, the file will be save * to <file>-<counter> where counter is the first integer that makes the file unique. * @param instances Instances to save */ private void saveWekaInstances(String file, Instances instances) { try { // Don't overwrite file. If a file with the given name exists, make a new version. if (FileIO.fileExists(file)) { int counter = 1; while (FileIO.fileExists(file + "-" + counter)) { counter++; } file = file + "-" + counter; } Log.INFO("Saving instances in: " + file); CSVSaver saver = new CSVSaver(); saver.setInstances(instances); saver.setFile(new File(file)); saver.writeBatch(); } catch (Exception e) { throw new RuntimeException(e); } }
From source file:meka.gui.explorer.classify.SaveCSV.java
License:Open Source License
/** * Returns the action lister to use in the menu. * * @param history the current history/* w w w . j a v a 2 s.c om*/ * @param index the selected history item * @return the listener */ @Override public ActionListener getActionListener(final ResultHistoryList history, final int index) { final Result result = history.getResultAt(index); return new ActionListener() { @Override public void actionPerformed(ActionEvent e) { Instances performance = (Instances) result .getMeasurement(IncrementalPerformance.RESULTS_SAMPLED_OVER_TIME); int retVal = getFileChooser().showSaveDialog(null); if (retVal != MekaFileChooser.APPROVE_OPTION) return; File file = getFileChooser().getSelectedFile(); try { CSVSaver saver = new CSVSaver(); saver.setInstances(performance); saver.setFile(getFileChooser().getSelectedFile()); saver.writeBatch(); } catch (Exception ex) { String msg = "Failed to write to '" + file + "'!"; System.err.println(msg); ex.printStackTrace(); JOptionPane.showMessageDialog(null, msg + "\n" + e); } } }; }
From source file:meka.gui.explorer.classify.SavePredictions.java
License:Open Source License
/** * Returns the action lister to use in the menu. * * @param history the current history//from w w w . j av a2 s .co m * @param index the selected history item * @return the listener */ @Override public ActionListener getActionListener(final ResultHistoryList history, final int index) { final Result result = history.getResultAt(index); return new ActionListener() { @Override public void actionPerformed(ActionEvent e) { int retVal = getFileChooser().showSaveDialog(null); if (retVal != MekaFileChooser.APPROVE_OPTION) return; File file = getFileChooser().getSelectedFile(); try { CSVSaver saver = new CSVSaver(); Instances performance = Result.getPredictionsAsInstances(result); saver.setInstances(performance); saver.setFile(getFileChooser().getSelectedFile()); saver.writeBatch(); } catch (Exception ex) { String msg = "Failed to write to '" + file + "'!"; System.err.println(msg); ex.printStackTrace(); JOptionPane.showMessageDialog(null, msg + "\n" + e); } } }; }