Example usage for org.apache.hadoop.io Text readString

Introduction

In this page you can find the example usage for org.apache.hadoop.io Text readString.

Prototype

public static String readString(DataInput in) throws IOException 

Source Link

Document

Read a UTF8 encoded string from in

Usage

From source file:org.apache.hama.bsp.TaskStatus.java

License:Apache License

@Override
public void readFields(DataInput in) throws IOException {
    this.jobId.readFields(in);
    this.taskId.readFields(in);
    this.progress = in.readFloat();
    this.runState = WritableUtils.readEnum(in, State.class);
    this.stateString = Text.readString(in);
    this.phase = WritableUtils.readEnum(in, Phase.class);
    this.startTime = in.readLong();
    this.finishTime = in.readLong();

    counters = new Counters();
    this.counters.readFields(in);
}

---

From source file:org.apache.hama.commons.io.PipesKeyValueWritable.java

License:Apache License

@Override
public void readFields(DataInput in) throws IOException {
    String str = Text.readString(in);
    // LOG.debug("readFields: '" + str + "'");

    this.keyValueDelimiter = str.charAt(0);
    str = str.substring(1);//from w  ww  .  j  av a2s  . co m
    String[] result = str.split(String.valueOf(this.keyValueDelimiter), 2);
    super.setKey(new Text(result[0]));
    super.setValue(new Text(result[1]));
}

---

From source file:org.apache.hama.commons.io.PipesVectorWritable.java

License:Apache License

public static DoubleVector readVector(DataInput in) throws IOException {
    String str = Text.readString(in);
    // LOG.debug("readVector: '" + str + "'");

    String[] values = str.split(",");
    int len = values.length;
    DoubleVector vector = new DenseDoubleVector(len);
    for (int i = 0; i < len; i++) {
        vector.set(i, Double.parseDouble(values[i]));
    }/*from w  w  w  .j  a va  2 s.c o m*/
    return vector;
}

---

From source file:org.apache.hama.ipc.ConnectionHeader.java

License:Apache License

@Override
public void readFields(DataInput in) throws IOException {
    protocol = Text.readString(in);
    if (protocol.isEmpty()) {
        protocol = null;//  w  w w .  ja  va2s .  c o  m
    }

    boolean ugiUsernamePresent = in.readBoolean();
    if (ugiUsernamePresent) {
        String username = in.readUTF();
        boolean realUserNamePresent = in.readBoolean();
        if (realUserNamePresent) {
            String realUserName = in.readUTF();
            UserGroupInformation realUserUgi = UserGroupInformation.createRemoteUser(realUserName);
            ugi = UserGroupInformation.createProxyUser(username, realUserUgi);
        } else {
            ugi = UserGroupInformation.createRemoteUser(username);
        }
    } else {
        ugi = null;
    }
}

---

From source file:org.apache.hama.pipes.protocol.UplinkReader.java

License:Apache License

public void incrementCounter() throws IOException {
    LOG.debug("Got MessageType.INCREMENT_COUNTER");

    String group = Text.readString(this.inStream);
    String name = Text.readString(this.inStream);
    long amount = WritableUtils.readVLong(this.inStream);

    LOG.debug("Got MessageType.INCREMENT_COUNTER group: " + group + " name: " + name + " amount: " + amount);

    peer.incrementCounter(group, name, amount);

    WritableUtils.writeVInt(this.outStream, MessageType.INCREMENT_COUNTER.code);
    binProtocol.flush();/* w  ww .  j  a  va2s. c om*/
    LOG.debug("Responded MessageType.INCREMENT_COUNTER");
}

---

From source file:org.apache.hama.pipes.protocol.UplinkReader.java

License:Apache License

@SuppressWarnings("unchecked")
public void sendMessage() throws IOException, InstantiationException, IllegalAccessException {
    String peerName = Text.readString(this.inStream);

    M message = (M) ReflectionUtils.newInstance(
            (Class<? extends M>) conf.getClass(Constants.MESSAGE_CLASS, BytesWritable.class), conf);

    LOG.debug("Got MessageType.SEND_MSG peerName: " + peerName + " messageClass: "
            + message.getClass().getName());

    readObject(message);//from  ww  w .j a v a 2  s.co  m

    peer.send(peerName, message);

    WritableUtils.writeVInt(this.outStream, MessageType.SEND_MSG.code);
    binProtocol.flush();
    LOG.debug("Responded MessageType.SEND_MSG");

    LOG.debug("Sent message to peerName: " + peerName + " messageClass: " + message.getClass().getName()
            + " Message: " + ((message.toString().length() < 10) ? message.toString()
                    : message.toString().substring(0, 9) + "..."));
}

---

From source file:org.apache.hama.pipes.protocol.UplinkReader.java

License:Apache License

public void seqFileOpen() throws IOException {
    String path = Text.readString(this.inStream);
    // option - read = "r" or write = "w"
    String option = Text.readString(this.inStream);
    // key and value class stored in the SequenceFile
    String keyClass = Text.readString(this.inStream);
    String valueClass = Text.readString(this.inStream);
    LOG.debug("GOT MessageType.SEQFILE_OPEN - Path: " + path);
    LOG.debug("GOT MessageType.SEQFILE_OPEN - Option: " + option);
    LOG.debug("GOT MessageType.SEQFILE_OPEN - KeyClass: " + keyClass);
    LOG.debug("GOT MessageType.SEQFILE_OPEN - ValueClass: " + valueClass);

    int fileID = -1;

    if (option.equals("r")) {
        SequenceFile.Reader reader;
        try {// w ww.  j  ava2  s  .  c o m
            reader = new SequenceFile.Reader(fs, new Path(path), conf);

            if (reader.getKeyClassName().equals(keyClass) && reader.getValueClassName().equals(valueClass)) {
                // try to load key and value class
                Class<?> sequenceKeyClass = conf.getClassLoader().loadClass(keyClass);
                Class<?> sequenceValueClass = conf.getClassLoader().loadClass(valueClass);

                // try to instantiate key and value class
                Writable sequenceKeyWritable = (Writable) ReflectionUtils.newInstance(sequenceKeyClass, conf);
                Writable sequenceValueWritable = (Writable) ReflectionUtils.newInstance(sequenceValueClass,
                        conf);

                // put new fileID and key and value Writable instances into HashMap
                fileID = reader.hashCode();
                this.sequenceFileReaders.put(fileID,
                        new AbstractMap.SimpleEntry<SequenceFile.Reader, Entry<Writable, Writable>>(reader,
                                new AbstractMap.SimpleEntry<Writable, Writable>(sequenceKeyWritable,
                                        sequenceValueWritable)));

            } else { // keyClass or valueClass is wrong
                fileID = -1;
                if (!reader.getKeyClassName().equals(keyClass)) {
                    LOG.error("SEQFILE_OPEN - Wrong KeyClass: " + keyClass + " File KeyClass: "
                            + reader.getKeyClassName());
                } else {
                    LOG.error("SEQFILE_OPEN - Wrong ValueClass: " + valueClass + " File ValueClass: "
                            + reader.getValueClassName());
                }
            }

        } catch (IOException e) {
            LOG.error("SEQFILE_OPEN - " + e.getMessage());
            fileID = -1;
        } catch (ClassNotFoundException e) {
            LOG.error("SEQFILE_OPEN - " + e.getMessage());
            fileID = -1;
        }

    } else if (option.equals("w")) {
        SequenceFile.Writer writer;
        try {
            // SequenceFile.Writer has an exclusive lease for a file
            // No other client can write to this file until other Writer has
            // completed
            if (!this.sequenceFileWriterPaths.contains(path)) {

                // try to load key and value class
                Class<?> sequenceKeyClass = conf.getClassLoader().loadClass(keyClass);
                Class<?> sequenceValueClass = conf.getClassLoader().loadClass(valueClass);

                // try to instantiate key and value class
                Writable sequenceKeyWritable = (Writable) ReflectionUtils.newInstance(sequenceKeyClass, conf);
                Writable sequenceValueWritable = (Writable) ReflectionUtils.newInstance(sequenceValueClass,
                        conf);

                writer = new SequenceFile.Writer(fs, conf, new Path(path), sequenceKeyClass,
                        sequenceValueClass);

                // put new fileID and key and value Writable instances into HashMap
                fileID = writer.hashCode();
                this.sequenceFileWriters.put(fileID,
                        new AbstractMap.SimpleEntry<SequenceFile.Writer, Entry<Writable, Writable>>(writer,
                                new AbstractMap.SimpleEntry<Writable, Writable>(sequenceKeyWritable,
                                        sequenceValueWritable)));

                // add path to set (exclusive access)
                this.sequenceFileWriterPaths.add(path);

            } else { // Path was already opened by another SequenceFile.Writer
                fileID = -1;
                LOG.error("SEQFILE_OPEN - Path: " + path + " is already used by another Writer!");
            }

        } catch (IOException e) {
            LOG.error("SEQFILE_OPEN - " + e.getMessage());
            fileID = -1;
        } catch (ClassNotFoundException e) {
            LOG.error("SEQFILE_OPEN - " + e.getMessage());
            fileID = -1;
        }
    } else { // wrong option
        LOG.error("SEQFILE_OPEN - Wrong option: '" + option + "'");
    }

    WritableUtils.writeVInt(this.outStream, MessageType.SEQFILE_OPEN.code);
    WritableUtils.writeVInt(this.outStream, fileID);
    binProtocol.flush();
    LOG.debug("Responded MessageType.SEQFILE_OPEN - FileID: " + fileID);
}

---

From source file:org.apache.ignite.internal.processors.hadoop.impl.v2.HadoopV2Job.java

License:Apache License

/** {@inheritDoc} */
@Override//from   w ww .java 2  s.  c om
public Collection<HadoopInputSplit> input() {
    ClassLoader oldLdr = HadoopCommonUtils.setContextClassLoader(jobConf.getClassLoader());

    try {
        String jobDirPath = jobConf.get(MRJobConfig.MAPREDUCE_JOB_DIR);

        if (jobDirPath == null) { // Probably job was submitted not by hadoop client.
            // Assume that we have needed classes and try to generate input splits ourself.
            if (jobConf.getUseNewMapper())
                return HadoopV2Splitter.splitJob(jobCtx);
            else
                return HadoopV1Splitter.splitJob(jobConf);
        }

        Path jobDir = new Path(jobDirPath);

        try {
            FileSystem fs = fileSystem(jobDir.toUri(), jobConf);

            JobSplit.TaskSplitMetaInfo[] metaInfos = SplitMetaInfoReader.readSplitMetaInfo(hadoopJobID, fs,
                    jobConf, jobDir);

            if (F.isEmpty(metaInfos))
                throw new IgniteCheckedException("No input splits found.");

            Path splitsFile = JobSubmissionFiles.getJobSplitFile(jobDir);

            try (FSDataInputStream in = fs.open(splitsFile)) {
                Collection<HadoopInputSplit> res = new ArrayList<>(metaInfos.length);

                for (JobSplit.TaskSplitMetaInfo metaInfo : metaInfos) {
                    long off = metaInfo.getStartOffset();

                    String[] hosts = metaInfo.getLocations();

                    in.seek(off);

                    String clsName = Text.readString(in);

                    HadoopFileBlock block = HadoopV1Splitter.readFileBlock(clsName, in, hosts);

                    if (block == null)
                        block = HadoopV2Splitter.readFileBlock(clsName, in, hosts);

                    res.add(block != null ? block : new HadoopExternalSplit(hosts, off));
                }

                return res;
            }
        } catch (Throwable e) {
            if (e instanceof Error)
                throw (Error) e;
            else
                throw transformException(e);
        }
    } catch (IgniteCheckedException e) {
        throw new IgniteException(e);
    } finally {
        HadoopCommonUtils.restoreContextClassLoader(oldLdr);
    }
}

---

From source file:org.apache.ignite.internal.processors.hadoop.impl.v2.HadoopV2TaskContext.java

License:Apache License

/**
 * @param split External split./*w  ww. j a  va 2 s .c  o  m*/
 * @return Native input split.
 * @throws IgniteCheckedException If failed.
 */
@SuppressWarnings("unchecked")
private Object readExternalSplit(HadoopExternalSplit split) throws IgniteCheckedException {
    Path jobDir = new Path(jobConf().get(MRJobConfig.MAPREDUCE_JOB_DIR));

    FileSystem fs;

    try {
        fs = fileSystemForMrUserWithCaching(jobDir.toUri(), jobConf(), fsMap);
    } catch (IOException e) {
        throw new IgniteCheckedException(e);
    }

    try (FSDataInputStream in = fs.open(JobSubmissionFiles.getJobSplitFile(jobDir))) {

        in.seek(split.offset());

        String clsName = Text.readString(in);

        Class<?> cls = jobConf().getClassByName(clsName);

        assert cls != null;

        Serialization serialization = new SerializationFactory(jobConf()).getSerialization(cls);

        Deserializer deserializer = serialization.getDeserializer(cls);

        deserializer.open(in);

        Object res = deserializer.deserialize(null);

        deserializer.close();

        assert res != null;

        return res;
    } catch (IOException | ClassNotFoundException e) {
        throw new IgniteCheckedException(e);
    }
}

---

From source file:org.apache.ignite.internal.processors.hadoop.v2.GridHadoopV2Job.java

License:Apache License

/** {@inheritDoc} */
@Override/* w w w .  j a va  2s. c o  m*/
public Collection<GridHadoopInputSplit> input() throws IgniteCheckedException {
    Thread.currentThread().setContextClassLoader(jobConf.getClassLoader());

    try {
        String jobDirPath = jobConf.get(MRJobConfig.MAPREDUCE_JOB_DIR);

        if (jobDirPath == null) { // Probably job was submitted not by hadoop client.
            // Assume that we have needed classes and try to generate input splits ourself.
            if (jobConf.getUseNewMapper())
                return GridHadoopV2Splitter.splitJob(jobCtx);
            else
                return GridHadoopV1Splitter.splitJob(jobConf);
        }

        Path jobDir = new Path(jobDirPath);

        try (FileSystem fs = FileSystem.get(jobDir.toUri(), jobConf)) {
            JobSplit.TaskSplitMetaInfo[] metaInfos = SplitMetaInfoReader.readSplitMetaInfo(hadoopJobID, fs,
                    jobConf, jobDir);

            if (F.isEmpty(metaInfos))
                throw new IgniteCheckedException("No input splits found.");

            Path splitsFile = JobSubmissionFiles.getJobSplitFile(jobDir);

            try (FSDataInputStream in = fs.open(splitsFile)) {
                Collection<GridHadoopInputSplit> res = new ArrayList<>(metaInfos.length);

                for (JobSplit.TaskSplitMetaInfo metaInfo : metaInfos) {
                    long off = metaInfo.getStartOffset();

                    String[] hosts = metaInfo.getLocations();

                    in.seek(off);

                    String clsName = Text.readString(in);

                    GridHadoopFileBlock block = GridHadoopV1Splitter.readFileBlock(clsName, in, hosts);

                    if (block == null)
                        block = GridHadoopV2Splitter.readFileBlock(clsName, in, hosts);

                    res.add(block != null ? block : new GridHadoopExternalSplit(hosts, off));
                }

                return res;
            }
        } catch (Throwable e) {
            throw transformException(e);
        }
    } finally {
        Thread.currentThread().setContextClassLoader(null);
    }
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• »