List of usage examples for weka.core.converters CSVSaver CSVSaver
public CSVSaver()
From source file:bi_project2.Classify.java
public Classify() { saver = new CSVSaver(); smo = new SMOreg(); // create a new SMO classifier-create }
From source file:cn.ict.zyq.bestConf.util.DataIOFile.java
License:Open Source License
/** * Save @param data to the CSV file at @param path *///from w w w .jav a 2 s .c o m public static void saveDataToCsvFile(String path, Instances data) throws IOException { System.out.println("\nSaving to file " + path + "..."); CSVSaver saver = new CSVSaver(); saver.setInstances(data); saver.setFile(new File(path)); saver.writeBatch(); }
From source file:com.kdcloud.engine.embedded.node.ConsoleOutput.java
License:Open Source License
@Override public void run() throws Exception { CSVSaver saver = new CSVSaver(); saver.setInstances(input.getInstances()); saver.setDestination(System.out); saver.writeBatch();/*from www .j av a 2 s . c o m*/ }
From source file:com.kdcloud.lib.rest.ext.InstancesRepresentation.java
License:Open Source License
protected Saver getSaver() throws IOException { if (getMediaType().equals(MediaType.TEXT_PLAIN)) return new ArffSaver(); else if (getMediaType().equals(MediaType.TEXT_CSV)) return new CSVSaver(); else if (getMediaType().equals(MediaType.APPLICATION_JSON)) return new JSONSaver(); else//from ww w . j av a 2 s .c o m raiseNotSupportedMediaType(); return null; }
From source file:com.sliit.neuralnetwork.RecurrentNN.java
public String trainModel(String modelName, String filePath, int outputs, int inputsTot) throws NeuralException { System.out.println("calling trainModel"); try {/*from ww w .j ava 2s . co m*/ System.out.println("Neural Network Training start"); loadSaveNN(modelName, false); if (model == null) { buildModel(); } File fileGeneral = new File(filePath); CSVLoader loader = new CSVLoader(); loader.setSource(fileGeneral); Instances instances = loader.getDataSet(); instances.setClassIndex(instances.numAttributes() - 1); StratifiedRemoveFolds stratified = new StratifiedRemoveFolds(); String[] options = new String[6]; options[0] = "-N"; options[1] = Integer.toString(5); options[2] = "-F"; options[3] = Integer.toString(1); options[4] = "-S"; options[5] = Integer.toString(1); stratified.setOptions(options); stratified.setInputFormat(instances); stratified.setInvertSelection(false); Instances testInstances = Filter.useFilter(instances, stratified); stratified.setInvertSelection(true); Instances trainInstances = Filter.useFilter(instances, stratified); String directory = fileGeneral.getParent(); CSVSaver saver = new CSVSaver(); File trainFile = new File(directory + "/" + "normtrainadded.csv"); File testFile = new File(directory + "/" + "normtestadded.csv"); if (trainFile.exists()) { trainFile.delete(); } trainFile.createNewFile(); if (testFile.exists()) { testFile.delete(); } testFile.createNewFile(); saver.setFile(trainFile); saver.setInstances(trainInstances); saver.writeBatch(); saver = new CSVSaver(); saver.setFile(testFile); saver.setInstances(testInstances); saver.writeBatch(); SequenceRecordReader recordReader = new CSVSequenceRecordReader(0, ","); recordReader.initialize(new org.datavec.api.split.FileSplit(trainFile)); SequenceRecordReader testReader = new CSVSequenceRecordReader(0, ","); testReader.initialize(new org.datavec.api.split.FileSplit(testFile)); DataSetIterator iterator = new org.deeplearning4j.datasets.datavec.SequenceRecordReaderDataSetIterator( recordReader, 2, outputs, inputsTot, false); DataSetIterator testIterator = new org.deeplearning4j.datasets.datavec.SequenceRecordReaderDataSetIterator( testReader, 2, outputs, inputsTot, false); roc = new ArrayList<Map<String, Double>>(); String statMsg = ""; Evaluation evaluation; for (int i = 0; i < 100; i++) { if (i % 2 == 0) { model.fit(iterator); evaluation = model.evaluate(testIterator); } else { model.fit(testIterator); evaluation = model.evaluate(iterator); } Map<String, Double> map = new HashMap<String, Double>(); Map<Integer, Integer> falsePositives = evaluation.falsePositives(); Map<Integer, Integer> trueNegatives = evaluation.trueNegatives(); Map<Integer, Integer> truePositives = evaluation.truePositives(); Map<Integer, Integer> falseNegatives = evaluation.falseNegatives(); double fpr = falsePositives.get(1) / (falsePositives.get(1) + trueNegatives.get(1)); double tpr = truePositives.get(1) / (truePositives.get(1) + falseNegatives.get(1)); map.put("FPR", fpr); map.put("TPR", tpr); roc.add(map); statMsg = evaluation.stats(); iterator.reset(); testIterator.reset(); } loadSaveNN(modelName, true); System.out.println("ROC " + roc); return statMsg; } catch (Exception e) { e.printStackTrace(); System.out.println("Error ocuured while building neural netowrk :" + e.getMessage()); throw new NeuralException(e.getLocalizedMessage(), e); } }
From source file:com.sliit.normalize.NormalizeDataset.java
public String normalizeDataset() { System.out.println("start normalizing data"); String filePathOut = ""; try {/* ww w . j av a2 s . c o m*/ CSVLoader loader = new CSVLoader(); if (reducedDiemensionFile != null) { loader.setSource(reducedDiemensionFile); } else { if (tempFIle != null && tempFIle.exists()) { loader.setSource(tempFIle); } else { loader.setSource(csvFile); } } Instances dataInstance = loader.getDataSet(); Normalize normalize = new Normalize(); dataInstance.setClassIndex(dataInstance.numAttributes() - 1); normalize.setInputFormat(dataInstance); String directory = csvFile.getParent(); outputFile = new File(directory + "/" + "normalized" + csvFile.getName()); if (!outputFile.exists()) { outputFile.createNewFile(); } CSVSaver saver = new CSVSaver(); saver.setFile(outputFile); for (int i = 1; i < dataInstance.numInstances(); i++) { normalize.input(dataInstance.instance(i)); } normalize.batchFinished(); Instances outPut = new Instances(dataInstance, 0); for (int i = 1; i < dataInstance.numInstances(); i++) { outPut.add(normalize.output()); } Attribute attribute = dataInstance.attribute(outPut.numAttributes() - 1); for (int j = 0; j < attribute.numValues(); j++) { if (attribute.value(j).equals("normal.")) { outPut.renameAttributeValue(attribute, attribute.value(j), "0"); } else { outPut.renameAttributeValue(attribute, attribute.value(j), "1"); } } saver.setInstances(outPut); saver.writeBatch(); writeToNewFile(directory); filePathOut = directory + "norm" + csvFile.getName(); if (tempFIle != null) { tempFIle.delete(); } if (reducedDiemensionFile != null) { reducedDiemensionFile.delete(); } outputFile.delete(); } catch (IOException e) { log.error("Error occurred:" + e.getMessage()); } catch (Exception e) { log.error("Error occurred:" + e.getMessage()); } return filePathOut; }
From source file:com.sliit.normalize.NormalizeDataset.java
public boolean updateStringValues(Map<Integer, String> values) { System.out.println("updating String Values"); boolean status = false; try {//w ww .ja v a 2 s.com csv.setSource(csvFile); Instances dataInstance = csv.getDataSet(); for (int i = 0; i < dataInstance.numInstances(); i++) { if (values.containsKey(i)) { Attribute attribute = dataInstance.attribute(i); for (int j = 0; j < attribute.numValues(); j++) { dataInstance.renameAttributeValue(attribute, attribute.value(j), j + ""); } } } tempFIle = new File(csvFile.getParent() + "/temp.csv"); CSVSaver saver = new CSVSaver(); saver.setInstances(dataInstance); saver.setFile(tempFIle); saver.writeBatch(); } catch (IOException e) { log.error("Error occurred:" + e.getMessage()); } return status; }
From source file:es.bsc.autonomic.powermodeller.tools.classifiers.WekaWrapper.java
License:Apache License
public static DataSet processDataSet(DataSet ds, VariableParser parser) { String independent = ds.getIndependent(); if (independent == null) throw new WekaWrapperException("Independent variable is not set in dataset."); HashMap<String, String> expression_list = parser.getNewMetrics(); Instances data = convertDataSetToInstances(ds); try {/*from w w w. j a va 2s . co m*/ // Apply filters for all the new variables for (Map.Entry<String, String> entry : expression_list.entrySet()) { String key = entry.getKey(); String value = entry.getValue(); logger.debug("Generating new variable " + key + " as " + value); AddExpression add_filter = new AddExpression(); add_filter.setName(key); add_filter.setExpression(value); add_filter.setInputFormat(data); data = useFilter(data, add_filter); } } catch (Exception e) { logger.error("Error while processing new variables", e); throw new WekaWrapperException("Error while processing new variables"); } // Iterate over all the columns and keep only the ones contained in variables list List<String> variables = parser.getColumns(); // Append independent variable to the list of variables to keep variables.add(independent); // Remove unneeded attributes try { // it's important to iterate from last to first, because when we remove // an instance, the rest shifts by one position. for (int i = data.numAttributes() - 1; i >= 0; i--) { String n = data.attribute(i).name(); if (!variables.contains(data.attribute(i).name())) { logger.trace("Deleting unnecessary attribute " + data.attribute(i).name()); data.deleteAttributeAt(i); } } data.toString(); } catch (Exception e) { logger.error("Error while removing unneeded variables", e); throw new WekaWrapperException("Error while removing unneeded variables"); } // Convert Instances in csv and return the new DataSet String new_path = CoreConfiguration.getNewCSVFileName(); try { CSVSaver saver = new CSVSaver(); saver.setInstances(data); saver.setFile(new File(new_path)); saver.writeBatch(); } catch (Exception e) { logger.error("Error while removing unneeded variables", e); throw new WekaWrapperException("Error while removing unneeded variables"); } DataSet ret = new DataSet(new_path); ret.setIndependent(independent); return ret; }
From source file:es.upm.dit.gsi.barmas.dataset.utils.DatasetSplitter.java
License:Open Source License
/** * @param folds//from w ww.jav a2s . c o m * @param minAgents * @param maxAgents * @param originalDatasetPath * @param outputDir * @param scenario * @param logger */ public void splitDataset(int folds, int minAgents, int maxAgents, String originalDatasetPath, String outputDir, String scenario, Logger logger) { int ratioint = (int) ((1 / (double) folds) * 100); double roundedratio = ((double) ratioint) / 100; // Look for essentials List<String[]> essentials = this.getEssentials(originalDatasetPath, logger); for (int fold = 0; fold < folds; fold++) { String outputDirWithRatio = outputDir + "/" + roundedratio + "testRatio/iteration-" + fold; File dir = new File(outputDirWithRatio); if (!dir.exists() || !dir.isDirectory()) { dir.mkdirs(); } logger.finer("--> splitDataset()"); logger.fine("Creating experiment.info..."); try { Instances originalData = this.getDataFromCSV(originalDatasetPath); originalData.randomize(new Random()); originalData.stratify(folds); // TestDataSet Instances testData = originalData.testCV(folds, fold); CSVSaver saver = new CSVSaver(); ArffSaver arffsaver = new ArffSaver(); File file = new File(outputDirWithRatio + File.separator + "test-dataset.csv"); if (!file.exists()) { saver.resetOptions(); saver.setInstances(testData); saver.setFile(file); saver.writeBatch(); } file = new File(outputDirWithRatio + File.separator + "test-dataset.arff"); if (!file.exists()) { arffsaver.resetOptions(); arffsaver.setInstances(testData); arffsaver.setFile(file); arffsaver.writeBatch(); } // BayesCentralDataset Instances trainData = originalData.trainCV(folds, fold); file = new File(outputDirWithRatio + File.separator + "bayes-central-dataset.csv"); if (!file.exists()) { saver.resetOptions(); saver.setInstances(trainData); saver.setFile(file); saver.writeBatch(); this.copyFileUsingApacheCommonsIO(file, new File( outputDirWithRatio + File.separator + "bayes-central-dataset-noEssentials.csv"), logger); CsvWriter w = new CsvWriter(new FileWriter(file, true), ','); for (String[] essential : essentials) { w.writeRecord(essential); } w.close(); } file = new File(outputDirWithRatio + File.separator + "bayes-central-dataset.arff"); if (!file.exists()) { arffsaver.resetOptions(); arffsaver.setInstances(trainData); arffsaver.setFile(file); arffsaver.writeBatch(); this.copyFileUsingApacheCommonsIO(file, new File( outputDirWithRatio + File.separator + "bayes-central-dataset-noEssentials.arff"), logger); CsvWriter w = new CsvWriter(new FileWriter(file, true), ','); for (String[] essential : essentials) { w.writeRecord(essential); } w.close(); } // Agent datasets CsvReader csvreader = new CsvReader(new FileReader(new File(originalDatasetPath))); csvreader.readHeaders(); String[] headers = csvreader.getHeaders(); csvreader.close(); for (int agents = minAgents; agents <= maxAgents; agents++) { this.createExperimentInfoFile(folds, agents, originalDatasetPath, outputDirWithRatio, scenario, logger); HashMap<String, CsvWriter> writers = new HashMap<String, CsvWriter>(); String agentsDatasetsDir = outputDirWithRatio + File.separator + agents + "agents"; HashMap<String, CsvWriter> arffWriters = new HashMap<String, CsvWriter>(); File f = new File(agentsDatasetsDir); if (!f.isDirectory()) { f.mkdirs(); } Instances copy = new Instances(trainData); copy.delete(); for (int i = 0; i < agents; i++) { String fileName = agentsDatasetsDir + File.separator + "agent-" + i + "-dataset.csv"; file = new File(fileName); if (!file.exists()) { CsvWriter writer = new CsvWriter(new FileWriter(fileName), ','); writer.writeRecord(headers); writers.put("AGENT" + i, writer); } fileName = agentsDatasetsDir + File.separator + "agent-" + i + "-dataset.arff"; file = new File(fileName); if (!file.exists()) { arffsaver.resetOptions(); arffsaver.setInstances(copy); arffsaver.setFile(new File(fileName)); arffsaver.writeBatch(); CsvWriter arffwriter = new CsvWriter(new FileWriter(fileName, true), ','); arffWriters.put("AGENT" + i, arffwriter); } logger.fine("AGENT" + i + " dataset created in csv and arff formats."); } // Append essentials to all for (String[] essential : essentials) { for (CsvWriter wr : writers.values()) { wr.writeRecord(essential); } for (CsvWriter arffwr : arffWriters.values()) { arffwr.writeRecord(essential); } } int agentCounter = 0; for (int j = 0; j < trainData.numInstances(); j++) { Instance instance = trainData.instance(j); CsvWriter writer = writers.get("AGENT" + agentCounter); CsvWriter arffwriter = arffWriters.get("AGENT" + agentCounter); String[] row = new String[instance.numAttributes()]; for (int a = 0; a < instance.numAttributes(); a++) { row[a] = instance.stringValue(a); } if (writer != null) { writer.writeRecord(row); } if (arffwriter != null) { arffwriter.writeRecord(row); } agentCounter++; if (agentCounter == agents) { agentCounter = 0; } } for (CsvWriter wr : writers.values()) { wr.close(); } for (CsvWriter arffwr : arffWriters.values()) { arffwr.close(); } } } catch (Exception e) { logger.severe("Exception while splitting dataset. ->"); logger.severe(e.getMessage()); System.exit(1); } logger.finest("Dataset for fold " + fold + " created."); } logger.finer("<-- splitDataset()"); }
From source file:etc.aloe.Aloe.java
License:Open Source License
protected void saveInstances(Instances instances, File outputFile) { try {/*from www . ja va2s . c o m*/ CSVSaver saver = new CSVSaver(); saver.setFile(outputFile); saver.setInstances(instances); saver.writeBatch(); System.out.println("Saved instances to " + outputFile); } catch (IOException e) { System.err.println("Error writing instances to " + outputFile); System.err.println("\t" + e.getMessage()); } }