Example usage for org.apache.hadoop.io Text toString

Introduction

In this page you can find the example usage for org.apache.hadoop.io Text toString.

Prototype

@Override
public String toString() 

Source Link

Document

Convert text back to string

Usage

From source file:cereal.impl.StoreImplTest.java

License:Apache License

@Test
public void testRead() throws Exception {
    Simple msg = Simple.newBuilder().setBoolean(true).setInt(42).setByteStr(ByteString.copyFromUtf8("string"))
            .setDub(3.14159d).build();//from  w w w.  j  a  v a 2  s  .  com
    final Text row = new Text("read");

    TreeMap<Key, Value> entries = new TreeMap<>();
    entries.put(new Key(row.toString(), "", "dub"), new Value("3.14159".getBytes(UTF_8)));
    entries.put(new Key(row.toString(), "", "int"), new Value("42".getBytes(UTF_8)));
    entries.put(new Key(row.toString(), "", "boolean"), new Value("true".getBytes(UTF_8)));
    entries.put(new Key(row.toString(), "", "byte_str"), new Value("string".getBytes(UTF_8)));

    Scanner scanner = createMock(Scanner.class);
    expect(conn.createScanner(table, Authorizations.EMPTY)).andReturn(scanner);
    scanner.setRange(Range.exact(row));
    expect(scanner.iterator()).andReturn(entries.entrySet().iterator());

    replay(conn, scanner);

    Simple msgCopy = store.read(row, Simple.class);

    verify(conn, scanner);

    assertEquals(msg, msgCopy);
}

---

From source file:cienciaCelularMR.FernetMapper.java

@Override
public void map(KeyMcell key, Text value, Context output) throws IOException, InterruptedException {
    try {/*from  w w  w .  jav  a  2 s . c om*/

        System.out.println("Entro al Map");
        System.out.println("Key del map fernet: " + key.toString());

        System.out.println("Fernet empezo a leer y guardar archivo .dat");
        try (FSDataInputStream fis = FileSystem.get(output.getConfiguration())
                .open(new Path(value.toString()))) {
            File archivo = new File("entradaFernet.dat");
            try (FileOutputStream fos = new FileOutputStream(archivo)) {
                byte[] buf = new byte[1024];
                int bytesRead;
                while ((bytesRead = fis.read(buf)) > 0) {
                    fos.write(buf, 0, bytesRead);
                    fos.flush();
                    output.progress();
                }
                fos.close();
                fis.close();
            }
        }

        System.out.println("Fernet termino de leer y guardar archivo .dat");
        Process process = new ProcessBuilder("fernet.exe", "--mode=" + key.getModoFernet().toString(),
                "--config=fernet.cfg", "entradaFernet.dat").start();

        InputStream is = process.getInputStream();
        InputStreamReader isr = new InputStreamReader(is);
        BufferedReader br = new BufferedReader(isr);
        String line;

        System.out.println("Fernet is running");
        System.out.println("La key de mapper fernet es: " + key);
        String res = "";
        while ((line = br.readLine()) != null) {
            res = res.concat(line);
            output.progress();
        }

        if ("point".equals(key.getModoFernet().toString())) {
            System.out.println("Fernet es point");
            String salidaName = "salidaFernet-" + key.getIdUsuario() + "." + key.getSubIdUsuario() + ".txt";
            FSDataOutputStream fs = FileSystem.get(output.getConfiguration()).create(new Path(salidaName));

            File salidaFile = new File("out_point.txt");
            if (salidaFile.exists()) {

                byte[] buffer = readFileToByteArray(new File("out_point.txt"));
                FernetOutput salida = new FernetOutput();
                salida.setFileName(new Text("out_point.txt"));
                salida.setSubId(key.getIdUsuario());
                salida.setValue(new BytesWritable(buffer));
                output.write(key, salida);
            }
        } else {
            File dir = new File(".");
            FileFilter fileFilter = new WildcardFileFilter("multi_*.txt");
            File[] files = dir.listFiles(fileFilter);
            for (File file : files) {
                byte[] buffer = readFileToByteArray(new File(file.getName()));
                FernetOutput salida = new FernetOutput();
                salida.setFileName(new Text(file.getName()));
                salida.setSubId(key.getIdUsuario());
                salida.setValue(new BytesWritable(buffer));
                output.write(key, salida);
            }
        }

    } catch (Exception ex) {
        String salidaName = "errorMapper-" + key.getIdUsuario() + "." + key.getSubIdUsuario() + ".txt";
        FSDataOutputStream fs = FileSystem.get(output.getConfiguration()).create(new Path(salidaName));
        fs.write(new Byte("Error en Mapper FERnet"));
        fs.write(new Byte("\n"));
        fs.flush();
        fs.close();

        Logger.getLogger(FernetMapper.class.getName()).log(Level.SEVERE, null, ex);
        throw ex;
    }
}

---

From source file:cityhubtopten.Top10Mapper.java

public void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {

    String line = value.toString();

    String[] tokens = line.split(",");
    if (line.contains("Quality of Life")) {
        return;/* w ww .j av a2 s  . c om*/
    }

    double quality = Double.parseDouble(tokens[6]);

    ToRecordMap.put(new QualityOfLife(quality), new Text(value));

    Iterator<Entry<QualityOfLife, Text>> iter = ToRecordMap.entrySet().iterator();

    Entry<QualityOfLife, Text> entry = null;

    while (ToRecordMap.size() > 10) {

        entry = iter.next();

        iter.remove();

    }

}

---

From source file:cityhubtopten.Top10Reducer.java

public void reduce(NullWritable key, Iterable<Text> values, Context context)
        throws IOException, InterruptedException {

    for (Text value : values) {

        String line = value.toString();

        if (line.length() > 0) {

            String[] tokens = line.split(",");

            //split the data and fetch salary

            double quality = Double.parseDouble(tokens[6]);

            //insert salary as key and entire row as value

            //tree map sort the records based on salary

            ToRecordMap.put(new QualityOfLife(quality), new Text(value));

        }// ww w.  j  a v a  2s .c o  m

    }

    // If we have more than ten records, remove the one with the lowest sal

    // As this tree map is sorted in descending order, the user with

    // the lowest sal is the last key.

    Iterator<Entry<QualityOfLife, Text>> iter = ToRecordMap.entrySet().iterator();

    Entry<QualityOfLife, Text> entry = null;

    while (ToRecordMap.size() > 10) {

        entry = iter.next();

        iter.remove();

    }

    for (Text t : ToRecordMap.descendingMap().values()) {

        // Output our ten records to the file system with a null key

        context.write(NullWritable.get(), t);

    }

}

---

From source file:clustering.init.WordSepMapper.java

License:Apache License

/**
 * Read the input file, extract the commodity info, and split the words
 * in g_name and g_model.//from  w w  w .  j a  v  a 2  s . c  o m
 *
 * @param key   position
 * @param value entry_id@@g_no@@code_ts@@g_name@@[g_model][@@other_columns]
 *              {@inheritDoc}
 */
@Override
protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {

    String[] contents = value.toString().split(this.splitter);
    String name = replaceSynonyms(contents[3]);

    String nameAndModel = SepUtils.append(name) + "##";

    if (contents.length > 3) {
        String model = replaceSynonyms(contents[4]);
        nameAndModel += SepUtils.append(model);
    }

    this.outputKey.set(contents[0] + "@@" + contents[1]);
    this.outputValue.set(nameAndModel);
    context.write(this.outputKey, this.outputValue);
}

---

From source file:clustering.inverted_index.InvertedIndexReducer.java

License:Apache License

/**
 * @param key    term//from ww  w .  ja  v a2 s . c om
 * @param values group_id=tf-idf
 *               {@inheritDoc}
 */
@Override
public void reduce(Text key, Iterable<Text> values, Context context) throws IOException, InterruptedException {

    int id = count.incrementAndGet();

    StringBuilder sb = new StringBuilder();

    /* append the index of this term */
    for (Text value : values) {
        String[] idAndTFIDF = value.toString().split("=");
        double tf_idf = Double.valueOf(idAndTFIDF[1]);
        /* filter the small tf_idfs
        which contributes little to the final similarity */
        if (!(this.pruning && tf_idf < this.pruningThreshold)) {
            sb.append(idAndTFIDF[0]).append('=').append(this.decimalFormat.format(tf_idf)).append(',');
        }
    }
    try {
        sb.deleteCharAt(sb.length() - 1);
    } catch (RuntimeException e) {
        System.out.println(sb.toString());
    }

    this.outputKey.set(id + ":" + key.toString());
    this.outputValue.set(sb.toString());
    // termId:term \t group_id=tf-idf,...
    context.write(this.outputKey, this.outputValue);
}

---

From source file:clustering.inverted_index.NormalizerReducer.java

License:Apache License

/**
 * @param key    group_id//from   w  ww  . ja va2s. c  o m
 * @param values term=tf_idf
 *               {@inheritDoc}
 */
@Override
protected void reduce(Text key, Iterable<Text> values, Context context)
        throws IOException, InterruptedException {

    this.tf_idfs.clear();
    for (Text value : values) {
        String[] termAndTF_IDF = value.toString().split("=");
        this.tf_idfs.put(termAndTF_IDF[0], Double.valueOf(termAndTF_IDF[1]));
    }
    double sum = 0.0d;
    for (double tf_idf : this.tf_idfs.values()) {
        sum += Math.pow(tf_idf, 2);
    }
    final double sq_sum = Math.sqrt(sum);

    this.tf_idfs.replaceAll((k, v) -> v / sq_sum);

    for (Map.Entry<String, Double> entry : this.tf_idfs.entrySet()) {
        this.outputKey.set(entry.getKey());
        this.outputValue.set(key.toString() + "=" + entry.getValue().toString());
        // term \t group_id=normalized_tf_idf
        context.write(this.outputKey, this.outputValue);
    }
}

---

From source file:clustering.link_back.pre.AttachMapper.java

License:Apache License

/**
 * @param key   entry_id/*from w  w w .  ja v  a  2 s  .  com*/
 * @param value g_no \t code_ts \t g_name \t g_name \t g_model [\t else]
 *              {@inheritDoc}
 */
@Override
public void map(Text key, Text value, Context context) throws IOException, InterruptedException {

    String[] line = value.toString().split("\t");

    this.outputKey.set(key.toString() + "@@" + line[0]);
    StringBuilder sb = new StringBuilder();
    for (int i = 1; i < line.length; i++) {
        sb.append(line[i]).append('\t');
    }
    sb.deleteCharAt(sb.length() - 1);
    this.outputValue.set(sb.toString());
    // entry_id@@g_no \t g_name@@g_model
    context.write(this.outputKey, this.outputValue);
}

---

From source file:clustering.link_back.step1.JoinReducer.java

License:Apache License

/**
 * @param key    group_id, join_order//from  www.  ja v  a  2 s  .  c om
 * @param values cluster_id in mst result,
 *               or entry_id@@g_no::g_name##g_model in simhash intermediate result.
 *               {@inheritDoc}
 */
@Override
public void reduce(Step1KeyWritable key, Iterable<Text> values, Context context)
        throws IOException, InterruptedException {
    // called on every group of keys
    for (Text value : values) {
        if (key.getTag().get() == 1) {
            // mst result, value = cluster_id
            this.outputValue.set(Integer.valueOf(value.toString()));
        } else {
            String[] contents = value.toString().split("::");
            this.outputKey.set(contents[0]);
            // cluster_id, entry_id@@g_no::g_name##g_model
            context.write(this.outputKey, this.outputValue);
        }
    }
}

---

From source file:clustering.link_back.step1.SetKeyMapper.java

License:Apache License

/**
 * @param key   group_id//from  www  .  j ava  2  s  .  c  o m
 * @param value cluster_id in mst result,
 *             or entry_id@@g_no::g_name##g_model in simhash intermediate result.
 *              {@inheritDoc}
 */
@Override
public void map(Text key, Text value, Context context) throws IOException, InterruptedException {
    int joinKey = Integer.valueOf(key.toString());
    this.taggedKey.set(joinKey, this.joinOrder);
    // (group_id,join_order) \t cluster_id or entry_id@@g_no::g_name##g_model
    context.write(this.taggedKey, value);
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• 37
• 38
• 39
• 40
• 41
• 42
• 43
• 44
• 45
• 46
• 47
• 48
• 49
• 50
• »