Example usage for org.apache.hadoop.io BytesWritable BytesWritable

Introduction

In this page you can find the example usage for org.apache.hadoop.io BytesWritable BytesWritable.

Prototype

public BytesWritable() 

Source Link

Document

Create a zero-size sequence.

Usage

From source file:alluxio.hadoop.mapreduce.KeyValueRecordReader.java

License:Apache License

@Override
public void initialize(InputSplit split, TaskAttemptContext context) throws IOException, InterruptedException {
    try {//from  www. jav a  2 s. c o m
        mReader = KeyValuePartitionReader.Factory.create(((KeyValueInputSplit) split).getPartitionId());
        mKeyValuePairIterator = mReader.iterator();
        mNumVisitedKeyValuePairs = 0;
        mNumKeyValuePairs = mReader.size();
        mCurrentKey = new BytesWritable();
        mCurrentValue = new BytesWritable();
    } catch (AlluxioException e) {
        throw new IOException(e);
    }
}

---

From source file:azkaban.common.web.JsonSequenceFileViewer.java

License:Apache License

public void displaySequenceFile(SequenceFile.Reader reader, PrintWriter output, int startLine, int endLine)
        throws IOException {

    if (logger.isDebugEnabled())
        logger.debug("display json file");

    try {/*from w w w . j  av a2  s  .com*/
        BytesWritable keyWritable = new BytesWritable();
        BytesWritable valueWritable = new BytesWritable();
        Text keySchema = reader.getMetadata().get(new Text("key.schema"));
        Text valueSchema = reader.getMetadata().get(new Text("value.schema"));

        JsonTypeSerializer keySerializer = new JsonTypeSerializer(keySchema.toString());
        JsonTypeSerializer valueSerializer = new JsonTypeSerializer(valueSchema.toString());

        // skip lines before the start line
        for (int i = 1; i < startLine; i++)
            reader.next(keyWritable, valueWritable);

        // now actually output lines
        for (int i = startLine; i <= endLine; i++) {
            boolean readSomething = reader.next(keyWritable, valueWritable);
            if (!readSomething)
                break;
            output.write(safeToString(keySerializer.toObject(keyWritable.getBytes())));
            output.write("\t=>\t");
            output.write(safeToString(valueSerializer.toObject(valueWritable.getBytes())));
            output.write("\n");
            output.flush();
        }
    } finally {
        reader.close();
    }
}

---

From source file:azkaban.viewer.hdfs.JsonSequenceFileViewer.java

License:Apache License

public void displaySequenceFile(AzkabanSequenceFileReader.Reader reader, PrintWriter output, int startLine,
        int endLine) throws IOException {

    if (logger.isDebugEnabled()) {
        logger.debug("display json file");
    }// ww w .  j av a2  s  . c o  m

    BytesWritable keyWritable = new BytesWritable();
    BytesWritable valueWritable = new BytesWritable();
    Text keySchema = reader.getMetadata().get(new Text("key.schema"));
    Text valueSchema = reader.getMetadata().get(new Text("value.schema"));

    JsonTypeSerializer keySerializer = new JsonTypeSerializer(keySchema.toString());
    JsonTypeSerializer valueSerializer = new JsonTypeSerializer(valueSchema.toString());

    // skip lines before the start line
    for (int i = 1; i < startLine; i++) {
        reader.next(keyWritable, valueWritable);
    }

    // now actually output lines
    for (int i = startLine; i <= endLine; i++) {
        boolean readSomething = reader.next(keyWritable, valueWritable);
        if (!readSomething) {
            break;
        }
        output.write(safeToString(keySerializer.toObject(keyWritable.getBytes())));
        output.write("\t=>\t");
        output.write(safeToString(valueSerializer.toObject(valueWritable.getBytes())));
        output.write("\n");
        output.flush();
    }
}

---

From source file:bigsatgps.BigDataHandler.java

License:Open Source License

/**
 *
 * @param infile/*from  ww w.  j  av  a  2s.co m*/
 * @return
 * @throws Exception
 */
public String ImageToSequence(String infile) throws Exception {
    String log4jConfPath = "lib/log4j.properties";
    PropertyConfigurator.configure(log4jConfPath);
    confHadoop = new Configuration();
    confHadoop.addResource(new Path("/hadoop/projects/hadoop-1.0.4/conf/core-site.xml"));
    confHadoop.addResource(new Path("/hadoop/projects/hadoop-1.0.4/conf/hdfs-site.xml"));
    FileSystem fs = FileSystem.get(confHadoop);
    Path inPath = new Path(infile);
    String outfile = infile.substring(0, infile.indexOf(".")) + ".seq";
    Path outPath = new Path(outfile);
    System.out.println();
    System.out.println("Successfully created the sequencefile " + outfile);
    FSDataInputStream in = null;
    Text key = new Text();
    BytesWritable value = new BytesWritable();
    SequenceFile.Writer writer = null;
    try {
        in = fs.open(inPath);
        byte buffer[] = new byte[in.available()];
        in.read(buffer);
        writer = SequenceFile.createWriter(fs, confHadoop, outPath, key.getClass(), value.getClass());
        writer.append(new Text(inPath.getName()), new BytesWritable(buffer));
        IOUtils.closeStream(writer);
        return outfile;
    } catch (IOException e) {
        System.err.println("Exception MESSAGES = " + e.getMessage());
        IOUtils.closeStream(writer);
        return null;
    }
}

---

From source file:boa.datagen.SeqProjectCombiner.java

License:Apache License

public static void main(String[] args) throws IOException {
    Configuration conf = new Configuration();
    conf.set("fs.default.name", "hdfs://boa-njt/");
    FileSystem fileSystem = FileSystem.get(conf);
    String base = conf.get("fs.default.name", "");

    HashMap<String, String> sources = new HashMap<String, String>();
    HashSet<String> marks = new HashSet<String>();
    FileStatus[] files = fileSystem.listStatus(new Path(base + "tmprepcache/2015-07"));
    for (int i = 0; i < files.length; i++) {
        FileStatus file = files[i];/*from  w w  w  .  j a v  a2 s .co m*/
        String name = file.getPath().getName();
        if (name.startsWith("projects-") && name.endsWith(".seq")) {
            System.out.println("Reading file " + i + " in " + files.length + ": " + name);
            SequenceFile.Reader r = new SequenceFile.Reader(fileSystem, file.getPath(), conf);
            final Text key = new Text();
            final BytesWritable value = new BytesWritable();
            try {
                while (r.next(key, value)) {
                    String s = key.toString();
                    if (marks.contains(s))
                        continue;
                    Project p = Project
                            .parseFrom(CodedInputStream.newInstance(value.getBytes(), 0, value.getLength()));
                    if (p.getCodeRepositoriesCount() > 0 && p.getCodeRepositories(0).getRevisionsCount() > 0)
                        marks.add(s);
                    sources.put(s, name);
                }
            } catch (Exception e) {
                System.err.println(name);
                e.printStackTrace();
            }
            r.close();
        }
    }
    SequenceFile.Writer w = SequenceFile.createWriter(fileSystem, conf,
            new Path(base + "repcache/2015-07/projects.seq"), Text.class, BytesWritable.class);
    for (int i = 0; i < files.length; i++) {
        FileStatus file = files[i];
        String name = file.getPath().getName();
        if (name.startsWith("projects-") && name.endsWith(".seq")) {
            System.out.println("Reading file " + i + " in " + files.length + ": " + name);
            SequenceFile.Reader r = new SequenceFile.Reader(fileSystem, file.getPath(), conf);
            final Text key = new Text();
            final BytesWritable value = new BytesWritable();
            try {
                while (r.next(key, value)) {
                    String s = key.toString();
                    if (sources.get(s).equals(name))
                        w.append(key, value);
                }
            } catch (Exception e) {
                System.err.println(name);
                e.printStackTrace();
            }
            r.close();
        }
    }
    w.close();

    fileSystem.close();
}

---

From source file:boa.functions.BoaAstIntrinsics.java

License:Apache License

/**
 * Given a ChangedFile, return the AST for that file at that revision.
 * /*from  www .  j  av a  2 s.  c  o  m*/
 * @param f the ChangedFile to get a snapshot of the AST for
 * @return the AST, or an empty AST on any sort of error
 */
@SuppressWarnings("unchecked")
@FunctionSpec(name = "getast", returnType = "ASTRoot", formalParameters = { "ChangedFile" })
public static ASTRoot getast(final ChangedFile f) {
    // since we know only certain kinds have ASTs, filter before looking up
    final ChangedFile.FileKind kind = f.getKind();
    if (kind != ChangedFile.FileKind.SOURCE_JAVA_ERROR && kind != ChangedFile.FileKind.SOURCE_JAVA_JLS2
            && kind != ChangedFile.FileKind.SOURCE_JAVA_JLS3 && kind != ChangedFile.FileKind.SOURCE_JAVA_JLS4)
        return emptyAst;

    context.getCounter(AST_COUNTER.GETS_ATTEMPTED).increment(1);

    final String rowName = f.getKey() + "!!" + f.getName();

    if (map == null)
        openMap();

    try {
        final BytesWritable value = new BytesWritable();
        if (map.get(new Text(rowName), value) == null) {
            context.getCounter(AST_COUNTER.GETS_FAIL_MISSING).increment(1);
        } else {
            final CodedInputStream _stream = CodedInputStream.newInstance(value.getBytes(), 0,
                    value.getLength());
            // defaults to 64, really big ASTs require more
            _stream.setRecursionLimit(Integer.MAX_VALUE);
            final ASTRoot root = ASTRoot.parseFrom(_stream);
            context.getCounter(AST_COUNTER.GETS_SUCCEED).increment(1);
            return root;
        }
    } catch (final InvalidProtocolBufferException e) {
        e.printStackTrace();
        context.getCounter(AST_COUNTER.GETS_FAIL_BADPROTOBUF).increment(1);
    } catch (final IOException e) {
        e.printStackTrace();
        context.getCounter(AST_COUNTER.GETS_FAIL_MISSING).increment(1);
    } catch (final RuntimeException e) {
        e.printStackTrace();
        context.getCounter(AST_COUNTER.GETS_FAIL_MISSING).increment(1);
    } catch (final Error e) {
        e.printStackTrace();
        context.getCounter(AST_COUNTER.GETS_FAIL_BADPROTOBUF).increment(1);
    }

    System.err.println("error with ast: " + rowName);
    context.getCounter(AST_COUNTER.GETS_FAILED).increment(1);
    return emptyAst;
}

---

From source file:boa.functions.BoaAstIntrinsics.java

License:Apache License

/**
 * Given a ChangedFile, return the comments for that file at that revision.
 * //from  w ww.j  a v a2s .co m
 * @param f the ChangedFile to get a snapshot of the comments for
 * @return the comments list, or an empty list on any sort of error
 */
@FunctionSpec(name = "getcomments", returnType = "CommentsRoot", formalParameters = { "ChangedFile" })
public static CommentsRoot getcomments(final ChangedFile f) {
    // since we know only certain kinds have comments, filter before looking up
    final ChangedFile.FileKind kind = f.getKind();
    if (kind != ChangedFile.FileKind.SOURCE_JAVA_ERROR && kind != ChangedFile.FileKind.SOURCE_JAVA_JLS2
            && kind != ChangedFile.FileKind.SOURCE_JAVA_JLS3 && kind != ChangedFile.FileKind.SOURCE_JAVA_JLS4)
        return emptyComments;

    final String rowName = f.getKey() + "!!" + f.getName();

    if (commentsMap == null)
        openCommentMap();

    try {
        final BytesWritable value = new BytesWritable();
        if (commentsMap.get(new Text(rowName), value) != null) {
            final CodedInputStream _stream = CodedInputStream.newInstance(value.getBytes(), 0,
                    value.getLength());
            final CommentsRoot root = CommentsRoot.parseFrom(_stream);
            return root;
        }
    } catch (final InvalidProtocolBufferException e) {
        e.printStackTrace();
    } catch (final IOException e) {
        e.printStackTrace();
    } catch (final RuntimeException e) {
        e.printStackTrace();
    } catch (final Error e) {
        e.printStackTrace();
    }

    System.err.println("error with comments: " + rowName);
    return emptyComments;
}

---

From source file:boa.functions.BoaAstIntrinsics.java

License:Apache License

/**
 * Given an IssueRepository, return the issues.
 * //from w w  w  .jav  a2s .c  o m
 * @param f the IssueRepository to get issues for
 * @return the issues list, or an empty list on any sort of error
 */
@FunctionSpec(name = "getissues", returnType = "IssuesRoot", formalParameters = { "IssueRepository" })
public static IssuesRoot getissues(final IssueRepository f) {
    if (issuesMap == null)
        openIssuesMap();

    try {
        final BytesWritable value = new BytesWritable();
        if (issuesMap.get(new Text(f.getKey()), value) != null) {
            final CodedInputStream _stream = CodedInputStream.newInstance(value.getBytes(), 0,
                    value.getLength());
            final IssuesRoot root = IssuesRoot.parseFrom(_stream);
            return root;
        }
    } catch (final InvalidProtocolBufferException e) {
        e.printStackTrace();
    } catch (final IOException e) {
        e.printStackTrace();
    } catch (final RuntimeException e) {
        e.printStackTrace();
    } catch (final Error e) {
        e.printStackTrace();
    }

    System.err.println("error with issues: " + f.getKey());
    return emptyIssues;
}

---

From source file:ca.sparkera.adapters.mapred.MainframeVBRecordReader.java

License:Apache License

@Override
public BytesWritable createValue() {
    return new BytesWritable();
}

---

From source file:cascading.avro.CascadingToAvroTest.java

License:Apache License

@Test
public void testToAvroFixedUsesValidRangeOfBytesWritable() {
    Schema fieldSchema = schema.getField("aFixed").schema();
    BytesWritable bytes = new BytesWritable();
    byte[] old_buffer_value = { 0, 1, 2, 3 };
    bytes.set(old_buffer_value, 0, old_buffer_value.length);

    byte[] buffer_value = { 4, 5, 6 };
    bytes.set(buffer_value, 0, buffer_value.length);
    byte[] outBytes = ((Fixed) CascadingToAvro.toAvroFixed(bytes, fieldSchema)).bytes();

    assertThat(outBytes, is(buffer_value));
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• »