Example usage for org.apache.hadoop.fs FSDataOutputStream hsync

List of usage examples for org.apache.hadoop.fs FSDataOutputStream hsync

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.fs.*
  5. FSDataOutputStream
  6. hsync

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FSDataOutputStream hsync.

Prototype

@Override 
    public void hsync() throws IOException

Source Link

Usage

From source file:com.continuent.tungsten.common.file.HdfsFileIO.java

License:Open Source License

/**
 * {@inheritDoc}/*from  w  w  w . j  a v  a  2  s  . co  m*/
 * 
 * @see com.continuent.tungsten.common.file.FileIO#write(com.continuent.tungsten.common.file.FilePath,
 *      java.lang.String, java.lang.String, boolean)
 */

@Override
public void write(FilePath path, String value, String charset, boolean fsync) throws FileIOException {
    // Write the data and flush to storage. This overwrites any
    // previous version.
    Path p = new Path(path.toString());
    FSDataOutputStream os = null;
    try {
        os = (FSDataOutputStream) this.getOutputStream(path);
        BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(os, charset));
        bw.write(value);
        bw.flush();
        if (fsync) {
            os.hsync();
        }
    } catch (IOException e) {
        throw new FileIOException("Unable to write data to file: uri=" + uri + " path=" + p.toString()
                + " value=" + safeSynopsis(value, 20), e);
    } finally {
        if (os != null) {
            try {
                os.close();
            } catch (IOException e) {
            }
        }
    }
}

From source file:com.streamsets.pipeline.stage.origin.hdfs.cluster.ClusterHDFSSourceIT.java

License:Apache License

private static void writeFile(FileSystem fs, Path ph, int size) throws IOException {
    FSDataOutputStream stm = fs.create(ph, true, 4096, (short) 3, 512);
    for (int i = 0; i < 1; i++) {
        stm.write(new byte[size]);
    }/*from www  . j av a  2  s.  c  om*/
    stm.hsync();
    stm.hsync();
    stm.close();
}

From source file:com.uber.hoodie.common.model.HoodiePartitionMetadata.java

License:Apache License

/**
 * Write the metadata safely into partition atomically.
 *//*from www  . ja v  a2  s .  co m*/
public void trySave(int taskPartitionId) {
    Path tmpMetaPath = new Path(partitionPath,
            HoodiePartitionMetadata.HOODIE_PARTITION_METAFILE + "_" + taskPartitionId);
    Path metaPath = new Path(partitionPath, HoodiePartitionMetadata.HOODIE_PARTITION_METAFILE);
    boolean metafileExists = false;

    try {
        metafileExists = fs.exists(metaPath);
        if (!metafileExists) {
            // write to temporary file
            FSDataOutputStream os = fs.create(tmpMetaPath, true);
            props.store(os, "partition metadata");
            os.hsync();
            os.hflush();
            os.close();

            // move to actual path
            fs.rename(tmpMetaPath, metaPath);
        }
    } catch (IOException ioe) {
        log.warn("Error trying to save partition metadata (this is okay, as long as "
                + "atleast 1 of these succced), " + partitionPath, ioe);
    } finally {
        if (!metafileExists) {
            try {
                // clean up tmp file, if still lying around
                if (fs.exists(tmpMetaPath)) {
                    fs.delete(tmpMetaPath, false);
                }
            } catch (IOException ioe) {
                log.warn("Error trying to clean up temporary files for " + partitionPath, ioe);
            }
        }
    }
}

From source file:edu.iu.wdamds.DataGen.java

License:Apache License

/**
 * Partition file generation is very important.
 * We need to verify it later. The format in
 * each partition file is: #ROWS Row ID ...
 * #FILES Distance File Weight File V File
 * /*from   w w w  .  j a v  a  2s .co m*/
 * @param inputFolder
 * @param inputPrefix
 * @param weightPrefix
 * @param vPrefix
 * @param workDirName
 * @param fs
 * @param numMapTasks
 * @return
 */
public static boolean generatePartitionFiles(String inputFolder, String inputPrefix, String weightPrefix,
        String vPrefix, Path dataDirPath, FileSystem fs, int numMapTasks) {
    // Assume inputFolder is a place where all
    // processes can access
    // (on some shared file system such as NFS)
    File inputDir = new File(inputFolder);
    if (!inputDir.exists() || !inputDir.isDirectory()) {
        return false;
    }
    String inputDirPath = inputDir.getAbsolutePath();
    File[] files = inputDir.listFiles();
    // The number of files should be even
    // One data file matches with one weight file
    // and one v file
    if (files.length % 3 != 0) {
        return false;
    }
    int numFileSets = files.length / 3;
    System.out.println("Number of file sets. " + numFileSets);
    int numSetPerPartition = numFileSets / numMapTasks;
    System.out.println("Number of file set per partition. " + numSetPerPartition);
    int restPartitions = numFileSets % numMapTasks;
    int partitionID = 0;
    int count = 0;
    int countLimit = numSetPerPartition;
    LinkedList<Integer> rowIDs = new LinkedList<>();
    LinkedList<String> filePaths = new LinkedList<>();
    for (int i = 0; i < files.length; i++) {
        String fileName = files[i].getName();
        System.out.println("File_" + i + " " + fileName);
        if (fileName.startsWith(inputPrefix)) {
            // Find a set of files and create rowID
            int rowID = Integer.parseInt(fileName.replaceFirst(inputPrefix, ""));
            // String inputFileName = fileName;
            String weightFileName = fileName.replaceFirst(inputPrefix, weightPrefix);
            String vFileName = fileName.replaceFirst(inputPrefix, vPrefix);
            rowIDs.add(rowID);
            filePaths.add(inputDirPath + "/" + fileName);
            filePaths.add(inputDirPath + "/" + weightFileName);
            filePaths.add(inputDirPath + "/" + vFileName);
            count++;
            // The base count limit is
            // numSetPerPartition,
            // I adjust it if there is still
            // partitions left.
            if (count == numSetPerPartition) {
                if (restPartitions > 0) {
                    countLimit = numSetPerPartition + 1;
                    restPartitions--;
                } else {
                    countLimit = numSetPerPartition;
                }
            }
            if (count == countLimit) {
                // Create a partition file
                Path partitionFilePath = new Path(dataDirPath, "partition_" + partitionID);
                try {
                    FSDataOutputStream out = fs.create(partitionFilePath, true);
                    BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(out));
                    bw.write(MDSConstants.ROWS_TAG);
                    bw.newLine();
                    for (int j = 0; j < rowIDs.size(); j++) {
                        bw.write(rowIDs.get(j).intValue() + "");
                        bw.newLine();
                    }
                    bw.write(MDSConstants.FILES_TAG);
                    bw.newLine();
                    for (int j = 0; j < filePaths.size(); j++) {
                        bw.write(filePaths.get(j));
                        bw.newLine();
                    }
                    bw.flush();
                    out.hflush();
                    out.hsync();
                    bw.close();
                } catch (IOException e) {
                    e.printStackTrace();
                    return false;
                }
                // Reset row ID holder
                rowIDs.clear();
                // Reset file list holder
                filePaths.clear();
                // Reset count
                count = 0;
                // Reset count limit
                countLimit = numSetPerPartition;
                // Increase partition ID
                partitionID++;
            }
        }
    }
    return true;
}

From source file:net.arp7.HdfsPerfTest.WriteFile.java

License:Apache License

/**
 * Write a single file to HDFS.//w w  w.  j av a2s.  c o m
 *
 * @param file
 * @param fs
 * @param data
 * @param stats object to accumulate write stats.
 * @throws IOException
 * @throws InterruptedException
 */
private static void writeOneFile(final Path file, final FileSystem fs, final byte[] data,
        final FileIoStats stats) throws IOException, InterruptedException {

    final long startTime = System.nanoTime();
    final EnumSet<CreateFlag> createFlags = EnumSet.of(CREATE, OVERWRITE);
    if (params.isLazyPersist()) {
        createFlags.add(LAZY_PERSIST);
    }

    LOG.info("Writing file " + file.toString());
    final FSDataOutputStream os = fs.create(file, FsPermission.getFileDefault(), createFlags,
            Constants.BUFFER_SIZE, params.getReplication(), params.getBlockSize(), null);
    final long createEndTime = System.nanoTime();
    stats.addCreateTime(createEndTime - startTime);
    final boolean isThrottled = params.maxWriteBps() > 0;
    final long expectedIoTimeNs = (isThrottled ? (((long) data.length * 1_000_000_000) / params.maxWriteBps())
            : 0);

    try {
        long lastLoggedPercent = 0;
        long writeStartTime = System.nanoTime();
        for (long j = 0; j < params.getFileSize() / params.getIoSize(); ++j) {
            final long ioStartTimeNs = (isThrottled ? System.nanoTime() : 0);
            os.write(data, 0, data.length);

            if (params.isHsync()) {
                os.hsync();
            } else if (params.isHflush()) {
                os.hflush();
            }

            final long ioEndTimeNs = (isThrottled ? System.nanoTime() : 0);
            Utils.enforceThrottle(ioEndTimeNs - ioStartTimeNs, expectedIoTimeNs);

            if (LOG.isDebugEnabled()) {
                long percentWritten = (j * params.getIoSize() * 100) / params.getFileSize();
                if (percentWritten > lastLoggedPercent) {
                    LOG.debug("  >> Wrote " + j * params.getIoSize() + "/" + params.getFileSize() + " ["
                            + percentWritten + "%]");
                    lastLoggedPercent = percentWritten;
                }
            }
        }

        final long writeEndTime = System.nanoTime();
        stats.addWriteTime(writeEndTime - writeStartTime);
        stats.incrFilesWritten();
        stats.incrBytesWritten(params.getFileSize());
    } finally {
        final long closeStartTime = System.nanoTime();
        os.close();
        final long closeEndTime = System.nanoTime();
        stats.addCloseTime(closeEndTime - closeStartTime);
    }
}

From source file:org.apache.tajo.ha.HdfsServiceTracker.java

License:Apache License

/**
 * It will creates the following form string. It includes
 *
 * <pre>//from   w  w  w .j  ava2s . c  o  m
 * {CLIENT_RPC_HOST:PORT}_{RESOURCE_TRACKER_HOST:PORT}_{CATALOG_HOST:PORT}_{MASTER_WEB_HOST:PORT}
 * </pre>
 *
 * @param isActive A boolean flag to indicate if it is for master or not.
 * @throws IOException
 */
private void createMasterFile(boolean isActive) throws IOException {
    String fileName = masterName.replaceAll(":", "_");
    Path path = null;

    if (isActive) {
        path = new Path(activePath, fileName);
    } else {
        path = new Path(backupPath, fileName);
    }

    StringBuilder sb = new StringBuilder();
    InetSocketAddress address = getHostAddress(HAConstants.MASTER_CLIENT_RPC_ADDRESS);
    sb.append(address.getAddress().getHostAddress()).append(":").append(address.getPort()).append("_");

    address = getHostAddress(HAConstants.RESOURCE_TRACKER_RPC_ADDRESS);
    sb.append(address.getAddress().getHostAddress()).append(":").append(address.getPort()).append("_");

    address = getHostAddress(HAConstants.CATALOG_ADDRESS);
    sb.append(address.getAddress().getHostAddress()).append(":").append(address.getPort()).append("_");

    address = getHostAddress(HAConstants.MASTER_INFO_ADDRESS);
    sb.append(address.getAddress().getHostAddress()).append(":").append(address.getPort());

    FSDataOutputStream out = fs.create(path);

    try {
        out.writeUTF(sb.toString());
        out.hsync();
        out.close();
    } catch (FileAlreadyExistsException e) {
        createMasterFile(false);
    }

    if (isActive) {
        isActiveStatus = true;
    } else {
        isActiveStatus = false;
    }

    startPingChecker();
}

From source file:org.apache.tez.dag.app.RecoveryParser.java

License:Apache License

public RecoveredDAGData parseRecoveryData() throws IOException {
    Path previousAttemptRecoveryDataDir = getPreviousAttemptRecoveryDataDir();
    LOG.info("Using " + previousAttemptRecoveryDataDir.toString()
            + " for recovering data from previous attempt");
    if (!recoveryFS.exists(previousAttemptRecoveryDataDir)) {
        LOG.info("Nothing to recover as previous attempt data does not exist" + ", previousAttemptDir="
                + previousAttemptRecoveryDataDir.toString());
        createDataRecoveredFlagFile();//from  w  w  w .ja va2s  .co  m
        return null;
    }

    Path summaryPath = getSummaryPath(previousAttemptRecoveryDataDir);
    FSDataInputStream summaryStream = getSummaryStream(summaryPath);
    if (summaryStream == null) {
        LOG.info("Nothing to recover as summary file does not exist" + ", previousAttemptDir="
                + previousAttemptRecoveryDataDir.toString() + ", summaryPath=" + summaryPath.toString());
        createDataRecoveredFlagFile();
        return null;
    }

    Path newSummaryPath = getSummaryPath(currentAttemptRecoveryDataDir);
    FSDataOutputStream newSummaryStream = getSummaryOutputStream(newSummaryPath);

    FileStatus summaryFileStatus = recoveryFS.getFileStatus(summaryPath);
    LOG.info("Parsing summary file" + ", path=" + summaryPath.toString() + ", len=" + summaryFileStatus.getLen()
            + ", lastModTime=" + summaryFileStatus.getModificationTime());

    int dagCounter = 0;

    Map<TezDAGID, DAGSummaryData> dagSummaryDataMap = new HashMap<TezDAGID, DAGSummaryData>();
    while (true) {
        RecoveryProtos.SummaryEventProto proto;
        try {
            proto = RecoveryProtos.SummaryEventProto.parseDelimitedFrom(summaryStream);
            if (proto == null) {
                LOG.info("Reached end of summary stream");
                break;
            }
        } catch (EOFException eof) {
            LOG.info("Reached end of summary stream");
            break;
        }
        HistoryEventType eventType = HistoryEventType.values()[proto.getEventType()];
        if (LOG.isDebugEnabled()) {
            LOG.debug("[RECOVERY SUMMARY]" + " dagId=" + proto.getDagId() + ", timestamp="
                    + proto.getTimestamp() + ", event=" + eventType);
        }
        TezDAGID dagId = TezDAGID.fromString(proto.getDagId());
        if (dagCounter < dagId.getId()) {
            dagCounter = dagId.getId();
        }
        if (!dagSummaryDataMap.containsKey(dagId)) {
            dagSummaryDataMap.put(dagId, new DAGSummaryData(dagId));
        }
        dagSummaryDataMap.get(dagId).handleSummaryEvent(proto);
        proto.writeDelimitedTo(newSummaryStream);
    }
    summaryStream.close();
    newSummaryStream.hsync();
    newSummaryStream.close();

    // Set counter for next set of DAGs & update dagNames Set in DAGAppMaster
    dagAppMaster.setDAGCounter(dagCounter);
    for (DAGSummaryData dagSummaryData : dagSummaryDataMap.values()) {
        dagAppMaster.dagNames.add(dagSummaryData.dagName);
        dagAppMaster.dagIDs.add(dagSummaryData.dagId.toString());
    }

    DAGSummaryData lastInProgressDAGData = getLastCompletedOrInProgressDAG(dagSummaryDataMap);
    if (lastInProgressDAGData == null) {
        LOG.info("Nothing to recover as no uncompleted/completed DAGs found");
        return null;
    }
    TezDAGID lastInProgressDAG = lastInProgressDAGData.dagId;
    if (lastInProgressDAG == null) {
        LOG.info("Nothing to recover as no uncompleted/completed DAGs found");
        return null;
    }

    LOG.info("Checking if DAG is in recoverable state" + ", dagId=" + lastInProgressDAGData.dagId);

    final RecoveredDAGData recoveredDAGData = new RecoveredDAGData();
    if (lastInProgressDAGData.completed) {
        recoveredDAGData.isCompleted = true;
        recoveredDAGData.dagState = lastInProgressDAGData.dagState;
    }

    String nonRecoverableReason = isDAGRecoverable(lastInProgressDAGData);
    if (nonRecoverableReason != null) {
        LOG.warn("Found last inProgress DAG but not recoverable: " + lastInProgressDAGData);
        recoveredDAGData.nonRecoverable = true;
        recoveredDAGData.reason = nonRecoverableReason;
    }

    LOG.info("Trying to recover dag from recovery file" + ", dagId=" + lastInProgressDAG.toString()
            + ", dataDir=" + previousAttemptRecoveryDataDir + ", intoCurrentDir="
            + currentAttemptRecoveryDataDir);

    FSDataInputStream dagRecoveryStream = getDAGRecoveryStream(previousAttemptRecoveryDataDir,
            lastInProgressDAG);
    if (dagRecoveryStream == null) {
        // Could not find data to recover
        // Error out
        throw new IOException(
                "Could not find recovery data for last in progress DAG" + ", dagId=" + lastInProgressDAG);
    }

    LOG.info("Copying DAG data into Current Attempt directory" + ", filePath="
            + getDAGRecoveryFilePath(currentAttemptRecoveryDataDir, lastInProgressDAG));
    FSDataOutputStream newDAGRecoveryStream = getDAGRecoveryOutputStream(currentAttemptRecoveryDataDir,
            lastInProgressDAG);

    boolean skipAllOtherEvents = false;
    while (true) {
        HistoryEvent event;
        try {
            event = getNextEvent(dagRecoveryStream);
            if (event == null) {
                LOG.info("Reached end of dag recovery stream");
                break;
            }
        } catch (EOFException eof) {
            LOG.info("Reached end of dag recovery stream");
            break;
        } catch (IOException ioe) {
            LOG.warn("Corrupt data found when trying to read next event", ioe);
            break;
        }
        if (skipAllOtherEvents) {
            // hit an error - skip reading other events
            break;
        }
        HistoryEventType eventType = event.getEventType();
        switch (eventType) {
        case DAG_SUBMITTED: {
            DAGSubmittedEvent submittedEvent = (DAGSubmittedEvent) event;
            LOG.info("Recovering from event" + ", eventType=" + eventType + ", event=" + event.toString());
            recoveredDAGData.recoveredDAG = dagAppMaster.createDAG(submittedEvent.getDAGPlan(),
                    lastInProgressDAG);
            recoveredDAGData.cumulativeAdditionalResources = submittedEvent
                    .getCumulativeAdditionalLocalResources();
            recoveredDAGData.recoveredDagID = recoveredDAGData.recoveredDAG.getID();
            dagAppMaster.setCurrentDAG(recoveredDAGData.recoveredDAG);
            if (recoveredDAGData.nonRecoverable) {
                skipAllOtherEvents = true;
            }
            break;
        }
        case DAG_INITIALIZED: {
            LOG.info("Recovering from event" + ", eventType=" + eventType + ", event=" + event.toString());
            assert recoveredDAGData.recoveredDAG != null;
            recoveredDAGData.recoveredDAG.restoreFromEvent(event);
            break;
        }
        case DAG_STARTED: {
            LOG.info("Recovering from event" + ", eventType=" + eventType + ", event=" + event.toString());
            assert recoveredDAGData.recoveredDAG != null;
            recoveredDAGData.recoveredDAG.restoreFromEvent(event);
            break;
        }
        case DAG_COMMIT_STARTED: {
            LOG.info("Recovering from event" + ", eventType=" + eventType + ", event=" + event.toString());
            assert recoveredDAGData.recoveredDAG != null;
            recoveredDAGData.recoveredDAG.restoreFromEvent(event);
            break;
        }
        case VERTEX_GROUP_COMMIT_STARTED: {
            LOG.info("Recovering from event" + ", eventType=" + eventType + ", event=" + event.toString());
            assert recoveredDAGData.recoveredDAG != null;
            recoveredDAGData.recoveredDAG.restoreFromEvent(event);
            break;
        }
        case VERTEX_GROUP_COMMIT_FINISHED: {
            LOG.info("Recovering from event" + ", eventType=" + eventType + ", event=" + event.toString());
            assert recoveredDAGData.recoveredDAG != null;
            recoveredDAGData.recoveredDAG.restoreFromEvent(event);
            break;
        }
        case DAG_FINISHED: {
            LOG.info("Recovering from event" + ", eventType=" + eventType + ", event=" + event.toString());
            // If this is seen, nothing to recover
            assert recoveredDAGData.recoveredDAG != null;
            recoveredDAGData.recoveredDAG.restoreFromEvent(event);
            recoveredDAGData.isCompleted = true;
            recoveredDAGData.dagState = ((DAGFinishedEvent) event).getState();
            skipAllOtherEvents = true;
            break;
        }
        case CONTAINER_LAUNCHED: {
            // Nothing to do for now
            break;
        }
        case VERTEX_INITIALIZED: {
            LOG.info("Recovering from event" + ", eventType=" + eventType + ", event=" + event.toString());
            assert recoveredDAGData.recoveredDAG != null;
            VertexInitializedEvent vEvent = (VertexInitializedEvent) event;
            Vertex v = recoveredDAGData.recoveredDAG.getVertex(vEvent.getVertexID());
            v.restoreFromEvent(vEvent);
            break;
        }
        case VERTEX_STARTED: {
            LOG.info("Recovering from event" + ", eventType=" + eventType + ", event=" + event.toString());
            assert recoveredDAGData.recoveredDAG != null;
            VertexStartedEvent vEvent = (VertexStartedEvent) event;
            Vertex v = recoveredDAGData.recoveredDAG.getVertex(vEvent.getVertexID());
            v.restoreFromEvent(vEvent);
            break;
        }
        case VERTEX_PARALLELISM_UPDATED: {
            LOG.info("Recovering from event" + ", eventType=" + eventType + ", event=" + event.toString());
            assert recoveredDAGData.recoveredDAG != null;
            VertexParallelismUpdatedEvent vEvent = (VertexParallelismUpdatedEvent) event;
            Vertex v = recoveredDAGData.recoveredDAG.getVertex(vEvent.getVertexID());
            v.restoreFromEvent(vEvent);
            break;
        }
        case VERTEX_COMMIT_STARTED: {
            LOG.info("Recovering from event" + ", eventType=" + eventType + ", event=" + event.toString());
            assert recoveredDAGData.recoveredDAG != null;
            VertexCommitStartedEvent vEvent = (VertexCommitStartedEvent) event;
            Vertex v = recoveredDAGData.recoveredDAG.getVertex(vEvent.getVertexID());
            v.restoreFromEvent(vEvent);
            break;
        }
        case VERTEX_FINISHED: {
            LOG.info("Recovering from event" + ", eventType=" + eventType + ", event=" + event.toString());
            assert recoveredDAGData.recoveredDAG != null;
            VertexFinishedEvent vEvent = (VertexFinishedEvent) event;
            Vertex v = recoveredDAGData.recoveredDAG.getVertex(vEvent.getVertexID());
            v.restoreFromEvent(vEvent);
            break;
        }
        case TASK_STARTED: {
            LOG.info("Recovering from event" + ", eventType=" + eventType + ", event=" + event.toString());
            assert recoveredDAGData.recoveredDAG != null;
            TaskStartedEvent tEvent = (TaskStartedEvent) event;
            Task task = recoveredDAGData.recoveredDAG.getVertex(tEvent.getTaskID().getVertexID())
                    .getTask(tEvent.getTaskID());
            task.restoreFromEvent(tEvent);
            break;
        }
        case TASK_FINISHED: {
            LOG.info("Recovering from event" + ", eventType=" + eventType + ", event=" + event.toString());
            assert recoveredDAGData.recoveredDAG != null;
            TaskFinishedEvent tEvent = (TaskFinishedEvent) event;
            Task task = recoveredDAGData.recoveredDAG.getVertex(tEvent.getTaskID().getVertexID())
                    .getTask(tEvent.getTaskID());
            task.restoreFromEvent(tEvent);
            break;
        }
        case TASK_ATTEMPT_STARTED: {
            LOG.info("Recovering from event" + ", eventType=" + eventType + ", event=" + event.toString());
            assert recoveredDAGData.recoveredDAG != null;
            TaskAttemptStartedEvent tEvent = (TaskAttemptStartedEvent) event;
            Task task = recoveredDAGData.recoveredDAG
                    .getVertex(tEvent.getTaskAttemptID().getTaskID().getVertexID())
                    .getTask(tEvent.getTaskAttemptID().getTaskID());
            task.restoreFromEvent(tEvent);
            break;
        }
        case TASK_ATTEMPT_FINISHED: {
            LOG.info("Recovering from event" + ", eventType=" + eventType + ", event=" + event.toString());
            assert recoveredDAGData.recoveredDAG != null;
            TaskAttemptFinishedEvent tEvent = (TaskAttemptFinishedEvent) event;
            Task task = recoveredDAGData.recoveredDAG
                    .getVertex(tEvent.getTaskAttemptID().getTaskID().getVertexID())
                    .getTask(tEvent.getTaskAttemptID().getTaskID());
            task.restoreFromEvent(tEvent);
            break;
        }
        case VERTEX_DATA_MOVEMENT_EVENTS_GENERATED: {
            LOG.info("Recovering from event" + ", eventType=" + eventType + ", event=" + event.toString());
            assert recoveredDAGData.recoveredDAG != null;
            VertexRecoverableEventsGeneratedEvent vEvent = (VertexRecoverableEventsGeneratedEvent) event;
            Vertex v = recoveredDAGData.recoveredDAG.getVertex(vEvent.getVertexID());
            v.restoreFromEvent(vEvent);
            break;
        }
        default:
            throw new RuntimeException("Invalid data found, unknown event type " + eventType);
        }
        if (LOG.isDebugEnabled()) {
            LOG.debug("[DAG RECOVERY]" + " dagId=" + lastInProgressDAG + ", eventType=" + eventType + ", event="
                    + event.toString());
        }
        newDAGRecoveryStream.writeInt(eventType.ordinal());
        event.toProtoStream(newDAGRecoveryStream);
    }
    dagRecoveryStream.close();
    newDAGRecoveryStream.hsync();
    newDAGRecoveryStream.close();

    if (!recoveredDAGData.isCompleted && !recoveredDAGData.nonRecoverable) {
        if (lastInProgressDAGData.bufferedSummaryEvents != null
                && !lastInProgressDAGData.bufferedSummaryEvents.isEmpty()) {
            for (HistoryEvent bufferedEvent : lastInProgressDAGData.bufferedSummaryEvents) {
                assert recoveredDAGData.recoveredDAG != null;
                switch (bufferedEvent.getEventType()) {
                case VERTEX_GROUP_COMMIT_STARTED:
                    recoveredDAGData.recoveredDAG.restoreFromEvent(bufferedEvent);
                    break;
                case VERTEX_GROUP_COMMIT_FINISHED:
                    recoveredDAGData.recoveredDAG.restoreFromEvent(bufferedEvent);
                    break;
                case VERTEX_FINISHED:
                    VertexFinishedEvent vertexFinishedEvent = (VertexFinishedEvent) bufferedEvent;
                    Vertex vertex = recoveredDAGData.recoveredDAG.getVertex(vertexFinishedEvent.getVertexID());
                    if (vertex == null) {
                        recoveredDAGData.nonRecoverable = true;
                        recoveredDAGData.reason = "All state could not be recovered"
                                + ", vertex completed but events not flushed" + ", vertexId="
                                + vertexFinishedEvent.getVertexID();
                    } else {
                        vertex.restoreFromEvent(vertexFinishedEvent);
                    }
                    break;
                default:
                    throw new RuntimeException("Invalid data found in buffered summary events"
                            + ", unknown event type " + bufferedEvent.getEventType());
                }
            }
        }
    }

    LOG.info("Finished copying data from previous attempt into current attempt");
    createDataRecoveredFlagFile();

    return recoveredDAGData;
}

From source file:org.apache.tez.test.TestTezJobs.java

License:Apache License

private void generateOrderedWordCountInput(Path inputDir, FileSystem fs) throws IOException {
    Path dataPath1 = new Path(inputDir, "inPath1");
    Path dataPath2 = new Path(inputDir, "inPath2");

    FSDataOutputStream f1 = null;
    FSDataOutputStream f2 = null;//from  w  ww  .ja  va 2s .c o  m
    try {
        f1 = fs.create(dataPath1);
        f2 = fs.create(dataPath2);

        final String prefix = "a";
        for (int i = 1; i <= 10; ++i) {
            final String word = prefix + "_" + i;
            for (int j = 10; j >= i; --j) {
                LOG.info("Writing " + word + " to input files");
                f1.write(word.getBytes());
                f1.writeChars("\t");
                f2.write(word.getBytes());
                f2.writeChars("\t");
            }
        }
        f1.hsync();
        f2.hsync();
    } finally {
        if (f1 != null) {
            f1.close();
        }
        if (f2 != null) {
            f2.close();
        }
    }
}

From source file:org.deeplearning4j.AnimalModelByHdfsSparkCluster.java

License:Apache License

public static void saveModel(FileSystem fs, Model model) throws Exception {

    String json = null;//from   w  w  w.ja v a  2  s  .c  o  m
    if (model instanceof MultiLayerNetwork) {
        json = ((MultiLayerNetwork) model).getLayerWiseConfigurations().toJson();
    } else if (model instanceof ComputationGraph) {
        json = ((ComputationGraph) model).getConfiguration().toJson();
    }
    byte[] byts = json.getBytes();
    FSDataOutputStream out = fs.create(new Path(modelPath));
    out.write(byts);
    out.hsync();
    fs.close();

}

From source file:org.deeplearning4j.listeners.SparkScoreIterationListener.java

License:Apache License

@Override
public void iterationDone(Model model, int iteration, int epoch) {

    if (printIterations <= 0) {
        printIterations = 1;/*from   w w  w  .j a  v a  2s .c o  m*/
    }
    String newScore = "";
    if (iteration % printIterations == 0) {
        double score = model.score();
        newScore += "Score at iteration {" + iteration + "} is {" + score + "}";
        log.info(newScore);
    }
    FileSystem nfs = null;
    try {
        nfs = CommonUtils.openHdfsConnect();
        Path path = new Path(pathStr);
        //although using append function isn't best ways, but currently it still solve the score log existing or not
        FSDataOutputStream out = nfs.append(path);//. .create(path);
        out.write(newScore.getBytes());
        out.write("\n".getBytes());
        out.hsync();
        out.close();
        CommonUtils.closeHdfsConnect(nfs);
    } catch (RemoteException e) {
        if (nfs != null) {
            CommonUtils.closeHdfsConnect(nfs);
        }
    } catch (Exception e) {
        throw new RuntimeException(e);
    }

}