Example usage for org.apache.hadoop.io IOUtils closeStream

Introduction

In this page you can find the example usage for org.apache.hadoop.io IOUtils closeStream.

Prototype

public static void closeStream(java.io.Closeable stream)

Source Link

Document

Closes the stream ignoring Throwable .

Usage

From source file:ml.shifu.shifu.core.dtrain.nn.NNOutput.java

License:Apache License

private void writeEncogModelToFileSystem(double[] weights, Path out) {
    FSDataOutputStream fos = null;/*from   w  w w.  j a va2  s  . co m*/
    try {
        fos = FileSystem.get(new Configuration()).create(out);
        LOG.info("Writing results to {}", out);
        this.network.getFlat().setWeights(weights);
        if (out != null) {
            EncogDirectoryPersistence.saveObject(fos, this.network);
        }
    } catch (IOException e) {
        LOG.error("Error in writing output.", e);
    } finally {
        IOUtils.closeStream(fos);
    }
}

From source file:ml.shifu.shifu.core.dtrain.NNOutput.java

License:Apache License

@Override
public void postApplication(MasterContext<NNParams, NNParams> context) {
    IOUtils.closeStream(this.progressOutput);

    // for dry mode, we don't save models files.
    if (this.isDry) {
        return;/*from   www . ja v  a 2  s  .c o  m*/
    }

    if (optimizeddWeights != null) {
        Path out = new Path(context.getProps().getProperty(NNConstants.GUAGUA_NN_OUTPUT));
        writeModelWeightsToFileSystem(optimizeddWeights, out);
    }
}

From source file:ml.shifu.shifu.core.dtrain.NNOutput.java

License:Apache License

private void writeModelWeightsToFileSystem(double[] weights, Path out) {
    FSDataOutputStream fos = null;/* w  w w  . j  a va  2  s  .  c  o  m*/
    try {
        fos = FileSystem.get(new Configuration()).create(out);
        LOG.info("Writing results to {}", out.toString());
        this.network.getFlat().setWeights(weights);
        if (out != null) {
            EncogDirectoryPersistence.saveObject(fos, this.network);
        }
    } catch (IOException e) {
        LOG.error("Error in writing output.", e);
    } finally {
        IOUtils.closeStream(fos);
    }
}

From source file:ml.shifu.shifu.core.dtrain.wdl.BinaryWDLSerializer.java

License:Apache License

public static void save(ModelConfig modelConfig, WideAndDeep wideAndDeep, FileSystem fs, Path output)
        throws IOException {
    DataOutputStream dos = null;//from   w  ww  .  ja  v a  2  s . co m
    try {
        dos = new DataOutputStream(new GZIPOutputStream(fs.create(output)));
        // version
        dos.writeInt(CommonConstants.WDL_FORMAT_VERSION);
        dos.writeUTF(modelConfig.getAlgorithm());
        // Reserved two float field, one double field and one string field
        dos.writeFloat(0.0f);
        dos.writeFloat(0.0f);
        dos.writeDouble(0.0d);
        dos.writeUTF("Reserved field");

        PersistWideAndDeep.save(wideAndDeep, dos);
        dos.writeUTF(modelConfig.getNormalizeType().name());
        dos.writeDouble(modelConfig.getNormalizeStdDevCutOff());
    } finally {
        IOUtils.closeStream(dos);
    }
}

From source file:ml.shifu.shifu.core.dtrain.wdl.BinaryWDLSerializer.java

License:Apache License

public static void save(ModelConfig modelConfig, List<ColumnConfig> columnConfigList, WideAndDeep wideAndDeep,
        FileSystem fs, Path output) throws IOException {
    DataOutputStream fos = null;/*from   w  ww  .j a  v  a2 s . c o  m*/
    try {
        fos = new DataOutputStream(new GZIPOutputStream(fs.create(output)));

        // version
        fos.writeInt(CommonConstants.WDL_FORMAT_VERSION);
        // Reserved two float field, one double field and one string field
        fos.writeFloat(0.0f);
        fos.writeFloat(0.0f);
        fos.writeDouble(0.0d);
        fos.writeUTF("Reserved field");

        // write normStr
        String normStr = modelConfig.getNormalize().getNormType().toString();
        StringUtils.writeString(fos, normStr);

        // compute columns needed
        Map<Integer, String> columnIndexNameMapping = getIndexNameMapping(columnConfigList);

        // write column stats to output
        List<NNColumnStats> csList = new ArrayList<>();
        for (ColumnConfig cc : columnConfigList) {
            if (columnIndexNameMapping.containsKey(cc.getColumnNum())) {
                NNColumnStats cs = new NNColumnStats();
                cs.setCutoff(modelConfig.getNormalizeStdDevCutOff());
                cs.setColumnType(cc.getColumnType());
                cs.setMean(cc.getMean());
                cs.setStddev(cc.getStdDev());
                cs.setColumnNum(cc.getColumnNum());
                cs.setColumnName(cc.getColumnName());
                cs.setBinCategories(cc.getBinCategory());
                cs.setBinBoundaries(cc.getBinBoundary());
                cs.setBinPosRates(cc.getBinPosRate());
                cs.setBinCountWoes(cc.getBinCountWoe());
                cs.setBinWeightWoes(cc.getBinWeightedWoe());

                // TODO cache such computation
                double[] meanAndStdDev = Normalizer.calculateWoeMeanAndStdDev(cc, false);
                cs.setWoeMean(meanAndStdDev[0]);
                cs.setWoeStddev(meanAndStdDev[1]);
                double[] weightMeanAndStdDev = Normalizer.calculateWoeMeanAndStdDev(cc, true);
                cs.setWoeWgtMean(weightMeanAndStdDev[0]);
                cs.setWoeWgtStddev(weightMeanAndStdDev[1]);

                csList.add(cs);
            }
        }

        fos.writeInt(csList.size());
        for (NNColumnStats cs : csList) {
            cs.write(fos);
        }

        // persist WideAndDeep Model
        wideAndDeep.write(fos);
    } finally {
        IOUtils.closeStream(fos);
    }
}

From source file:ml.shifu.shifu.core.dtrain.wdl.WDLOutput.java

License:Apache License

@Override
public void postApplication(MasterContext<WDLParams, WDLParams> context) {
    Path out = new Path(context.getProps().getProperty(CommonConstants.GUAGUA_OUTPUT));
    writeModelToFileSystem(context.getMasterResult(), out);
    if (this.isGsMode || this.isKFoldCV) {
        Path valErrOutput = new Path(context.getProps().getProperty(CommonConstants.GS_VALIDATION_ERROR));
        double valErr = context.getMasterResult().getValidationError()
                / context.getMasterResult().getValidationCount();
        writeValErrorToFileSystem(valErr, valErrOutput);
    }//from w w w  .  java 2s. c o m
    IOUtils.closeStream(this.progressOutput);
}

From source file:ml.shifu.shifu.core.dvarsel.VarSelOutput.java

License:Apache License

private void writeColumnIdsIntoHDFS(String path, List<Integer> columnIds) {
    BufferedWriter bw = null;//from  ww  w .jav a  2s  .  c o m
    try {
        bw = ShifuFileUtils.getWriter(path, SourceType.HDFS);
        bw.write(String.format("%s|%s", Integer.toString(columnIds.size()), columnIds.toString()));
        bw.newLine();
        bw.flush();
    } catch (IOException e) {
        e.printStackTrace();
        LOG.error("Error in writing output.", e);
    } finally {
        IOUtils.closeStream(bw);
    }
}

From source file:mzb.NameNodeConnector.java

License:Apache License

/** Close the connection. */
void close() {/*  w  ww  .j  a va  2  s  .c o m*/
    shouldRun = false;
    try {
        if (keyupdaterthread != null) {
            keyupdaterthread.interrupt();
        }
    } catch (Exception e) {
        LOG.warn("Exception shutting down access key updater thread", e);
    }

    // close the output file
    IOUtils.closeStream(out);
    if (fs != null) {
        try {
            fs.delete(BALANCER_ID_PATH, true);
        } catch (IOException ioe) {
            LOG.warn("Failed to delete " + BALANCER_ID_PATH, ioe);
        }
    }
}

From source file:net.ufida.info.util.TasteHadoopUtils.java

License:Apache License

/**
 * Reads a text-based outputfile that only contains an int
 *///ww w  .  ja va  2 s . c o  m
public static int readIntFromFile(Configuration conf, Path outputDir) throws IOException {
    FileSystem fs = outputDir.getFileSystem(conf);
    Path outputFile = fs.listStatus(outputDir, PathFilters.partFilter())[0].getPath();
    InputStream in = null;
    try {
        in = fs.open(outputFile);
        ByteArrayOutputStream out = new ByteArrayOutputStream();
        IOUtils.copyBytes(in, out, conf);
        return Integer.parseInt(new String(out.toByteArray(), Charsets.UTF_8).trim());
    } finally {
        IOUtils.closeStream(in);
    }
}

From source file:nl.basjes.utils.TestWritableInterface.java

License:Apache License

/**
 * Converts an instance of Writable into a byte[].
 * @param writable// www  . j  ava2 s .  co m
 * @return
 * @throws IOException
 */
public static byte[] serialize(Writable writable) throws IOException {
    ByteArrayOutputStream out = new ByteArrayOutputStream();
    DataOutputStream dataOut = null;
    try {
        dataOut = new DataOutputStream(out);
        writable.write(dataOut);
        return out.toByteArray();
    } finally {
        IOUtils.closeStream(dataOut);
    }
}