Example usage for org.apache.hadoop.io IOUtils closeStream

Introduction

In this page you can find the example usage for org.apache.hadoop.io IOUtils closeStream.

Prototype

public static void closeStream(java.io.Closeable stream) 

Source Link

Document

Closes the stream ignoring Throwable .

Usage

From source file:ml.shifu.dtrain.NNOutput.java

License:Apache License

@Override
public void postApplication(MasterContext<NNParams, NNParams> context) {
    IOUtils.closeStream(this.progressOutput);

    // for dry mode, we don't save models files.
    if (this.isDry) {
        return;//from w ww  .  j a v a2s .  c o  m
    }

    if (optimizeddWeights != null) {
        Path out = new Path(context.getProps().getProperty(DtrainConstants.GUAGUA_NN_OUTPUT));
        writeModelWeightsToFileSystem(optimizeddWeights, out);
    }
}

---

From source file:ml.shifu.dtrain.NNOutput.java

License:Apache License

private void writeModelWeightsToFileSystem(double[] weights, Path out) {
    FSDataOutputStream fos = null;//  w  w  w .j ava 2 s  .co m
    try {
        // TODO - this workaround (..conf.addResource(..)) was
        // needed to work with HDP2.x. Find a better fix
        conf.addResource(new Path("/etc/hadoop/conf/core-site.xml"));
        conf.addResource(new Path("/etc/hadoop/conf/hdfs-site.xml"));

        fos = FileSystem.get(conf).create(out);
        LOG.info("Writing results to {}", out);
        this.network.getFlat().setWeights(weights);
        if (out != null) {
            EncogDirectoryPersistence.saveObject(fos, this.network);
        }
    } catch (IOException e) {
        LOG.error("Error in writing output.", e);
    } finally {
        IOUtils.closeStream(fos);
    }
}

---

From source file:ml.shifu.guagua.example.kmeans.KMeansCentriodsOutput.java

License:Apache License

@Override
public void postApplication(MasterContext<KMeansMasterParams, KMeansWorkerParams> context) {
    LOG.info("KMeansCentersOutput starts to write k centers to file.");

    Path out = new Path(context.getProps().getProperty(KMeansContants.KMEANS_CENTERS_OUTPUT));
    PrintWriter pw = null;//w w  w  . ja  v a  2s  .  c o m
    try {
        FSDataOutputStream fos = FileSystem.get(new Configuration()).create(out);
        LOG.info("Writing results to {}", out.toString());
        pw = new PrintWriter(fos);
        KMeansMasterParams masterResult = context.getMasterResult();
        for (double[] center : masterResult.getPointList()) {
            pw.println(Arrays.toString(center));
        }
        pw.flush();
    } catch (IOException e) {
        LOG.error("Error in writing output.", e);
    } finally {
        IOUtils.closeStream(pw);
    }
}

---

From source file:ml.shifu.guagua.example.kmeans.KMeansDataOutput.java

License:Apache License

@Override
public void postApplication(WorkerContext<KMeansMasterParams, KMeansWorkerParams> context) {
    Path outFolder = new Path(context.getProps().getProperty(KMeansContants.KMEANS_DATA_OUTPUT,
            "part-g-" + context.getContainerId()));
    String separator = context.getProps().getProperty(KMeansContants.KMEANS_DATA_SEPERATOR);

    @SuppressWarnings("unchecked")
    MemoryDiskList<TaggedRecord> dataList = (MemoryDiskList<TaggedRecord>) context.getAttachment();

    PrintWriter pw = null;/*from   w w  w . j a v  a 2  s  .c  om*/
    try {
        FileSystem fileSystem = FileSystem.get(new Configuration());
        fileSystem.mkdirs(outFolder);

        Path outputFile = new Path(outFolder, "part-g-" + context.getContainerId());
        FSDataOutputStream fos = fileSystem.create(outputFile);
        LOG.info("Writing results to {}", outputFile.toString());
        pw = new PrintWriter(fos);
        for (TaggedRecord record : dataList) {
            pw.println(record.toString(separator));
        }
        pw.flush();
    } catch (IOException e) {
        LOG.error("Error in writing output.", e);
    } finally {
        IOUtils.closeStream(pw);
    }
}

---

From source file:ml.shifu.guagua.example.lnr.LinearRegressionOutput.java

License:Apache License

/**
 * Get output file setting and write final sum value to HDFS file.
 *///www .  j a v  a  2  s .co  m
@Override
public void postApplication(final MasterContext<LinearRegressionParams, LinearRegressionParams> context) {
    LOG.info("Starts to write final value to file.");
    Path out = new Path(context.getProps().getProperty("lr.model.output"));
    LOG.info("Writing results to {}", out.toString());
    PrintWriter pw = null;
    try {
        FSDataOutputStream fos = FileSystem.get(new Configuration()).create(out);
        pw = new PrintWriter(fos);
        pw.println(Arrays.toString(context.getMasterResult().getParameters()));
        pw.flush();
    } catch (IOException e) {
        LOG.error("Error in writing output.", e);
    } finally {
        IOUtils.closeStream(pw);
    }
}

---

From source file:ml.shifu.guagua.example.lr.LogisticRegressionOutput.java

License:Apache License

/**
 * Get output file setting and write final sum value to HDFS file.
 *//*from w  w w.j av  a  2 s  .  co  m*/
@Override
public void postApplication(final MasterContext<LogisticRegressionParams, LogisticRegressionParams> context) {
    LOG.info("Starts to write final value to file.");
    Path out = new Path(context.getProps().getProperty("lr.model.output"));
    LOG.info("Writing results to {}", out.toString());
    PrintWriter pw = null;
    try {
        FSDataOutputStream fos = FileSystem.get(new Configuration()).create(out);
        pw = new PrintWriter(fos);
        pw.println(Arrays.toString(context.getMasterResult().getParameters()));
        pw.flush();
    } catch (IOException e) {
        LOG.error("Error in writing output.", e);
    } finally {
        IOUtils.closeStream(pw);
    }
}

---

From source file:ml.shifu.guagua.example.nn.NNOutput.java

License:Apache License

@Override
public void postApplication(MasterContext<NNParams, NNParams> context) {
    LOG.info("NNOutput starts to write model to files.");
    int inputs = NumberFormatUtils.getInt(context.getProps().getProperty(NNConstants.GUAGUA_NN_INPUT_NODES),
            NNConstants.GUAGUA_NN_DEFAULT_INPUT_NODES);
    int hiddens = NumberFormatUtils.getInt(context.getProps().getProperty(NNConstants.GUAGUA_NN_HIDDEN_NODES),
            NNConstants.GUAGUA_NN_DEFAULT_HIDDEN_NODES);
    int outputs = NumberFormatUtils.getInt(context.getProps().getProperty(NNConstants.GUAGUA_NN_OUTPUT_NODES),
            NNConstants.GUAGUA_NN_DEFAULT_OUTPUT_NODES);
    BasicNetwork network = NNUtils.generateNetwork(inputs, hiddens, outputs);

    Path out = new Path(context.getProps().getProperty(NNConstants.GUAGUA_NN_OUTPUT));
    FSDataOutputStream fos = null;/*from  ww w.ja  v  a  2 s .  co m*/
    try {
        fos = FileSystem.get(new Configuration()).create(out);
        LOG.info("Writing results to {}", out.toString());
        network.getFlat().setWeights(context.getMasterResult().getWeights());
        EncogDirectoryPersistence.saveObject(fos, network);
    } catch (IOException e) {
        LOG.error("Error in writing output.", e);
    } finally {
        IOUtils.closeStream(fos);
    }
}

---

From source file:ml.shifu.guagua.example.sum.SumOutput.java

License:Apache License

/**
 * Get output file setting and write final sum value to HDFS file.
 */// w  w  w.j  a v a2s .  c o m
@Override
public void postApplication(
        final MasterContext<GuaguaWritableAdapter<LongWritable>, GuaguaWritableAdapter<LongWritable>> context) {
    LOG.info("SumOutput starts to write final sum value to file.");
    Path out = new Path(context.getProps().getProperty("guagua.sum.output"));
    LOG.info("Writing results to {}", out.toString());
    PrintWriter pw = null;
    try {
        FSDataOutputStream fos = FileSystem.get(new Configuration()).create(out);
        pw = new PrintWriter(fos);
        pw.println(context.getMasterResult().getWritable().get());
        pw.flush();
    } catch (IOException e) {
        LOG.error("Error in writing output.", e);
    } catch (Exception e) {
        LOG.error("Error in writing output.", e);
    } finally {
        IOUtils.closeStream(pw);
    }
}

---

From source file:ml.shifu.guagua.hadoop.ZooKeeperMasterInterceptor.java

License:Apache License

/**
 * Write zookeeper server info to HDFS. Then worker can get such info and connect to such server..
 *///from   ww  w  . jav a  2  s .  c  o m
private void writeServerInfoToHDFS(MasterContext<MASTER_RESULT, WORKER_RESULT> context,
        String embededZooKeeperServer) {
    String hdfsZookeeperServerFolder = getZookeeperServerFolder(context);
    PrintWriter pw = null;
    try {
        if (!this.fileSystem.exists(new Path(hdfsZookeeperServerFolder))) {
            this.fileSystem.mkdirs(new Path(hdfsZookeeperServerFolder));
        }
        this.zookeeperServerPath = fileSystem.makeQualified(
                new Path(hdfsZookeeperServerFolder, GuaguaConstants.GUAGUA_CLUSTER_ZOOKEEPER_SERVER_FILE));
        LOG.info("Writing hdfs zookeeper server info to {}", this.zookeeperServerPath);
        FSDataOutputStream fos = fileSystem.create(this.zookeeperServerPath);
        pw = new PrintWriter(fos);
        pw.println(embededZooKeeperServer);
        pw.flush();
    } catch (IOException e) {
        LOG.error("Error in writing output.", e);
    } catch (Exception e) {
        LOG.error("Error in writing output.", e);
    } finally {
        IOUtils.closeStream(pw);
    }
}

---

From source file:ml.shifu.guagua.mapreduce.example.kmeans.KMeansDataOutput.java

License:Apache License

@Override
public void postApplication(WorkerContext<KMeansMasterParams, KMeansWorkerParams> context) {
    Path outFolder = new Path(context.getProps().getProperty(KMeansContants.KMEANS_DATA_OUTPUT,
            "part-g-" + context.getContainerId()));
    String separator = context.getProps().getProperty(KMeansContants.KMEANS_DATA_SEPERATOR);

    @SuppressWarnings("unchecked")
    List<TaggedRecord> dataList = (List<TaggedRecord>) context.getAttachment();

    PrintWriter pw = null;/*from  www .java2 s. c o  m*/
    try {
        FileSystem fileSystem = FileSystem.get(new Configuration());
        fileSystem.mkdirs(outFolder);

        Path outputFile = new Path(outFolder, "part-g-" + context.getContainerId());
        FSDataOutputStream fos = fileSystem.create(outputFile);
        LOG.info("Writing results to {}", outputFile.toString());
        pw = new PrintWriter(fos);
        for (TaggedRecord record : dataList) {
            pw.println(record.toString(separator));
        }
        pw.flush();
    } catch (IOException e) {
        LOG.error("Error in writing output.", e);
    } finally {
        IOUtils.closeStream(pw);
    }
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• »