Example usage for org.apache.hadoop.io Text Text

Introduction

In this page you can find the example usage for org.apache.hadoop.io Text Text.

Prototype

public Text(byte[] utf8) 

Source Link

Document

Construct from a byte array.

Usage

From source file:cereal.impl.StoreImplTest.java

License:Apache License

@Test(expected = NullPointerException.class)
public void testNullReadClass() throws Exception {
    store.read(new Text("id"), null);
}

---

From source file:cereal.impl.StoreImplTest.java

License:Apache License

@Test
public void testRead() throws Exception {
    Simple msg = Simple.newBuilder().setBoolean(true).setInt(42).setByteStr(ByteString.copyFromUtf8("string"))
            .setDub(3.14159d).build();//ww  w . j  a v a 2s . co m
    final Text row = new Text("read");

    TreeMap<Key, Value> entries = new TreeMap<>();
    entries.put(new Key(row.toString(), "", "dub"), new Value("3.14159".getBytes(UTF_8)));
    entries.put(new Key(row.toString(), "", "int"), new Value("42".getBytes(UTF_8)));
    entries.put(new Key(row.toString(), "", "boolean"), new Value("true".getBytes(UTF_8)));
    entries.put(new Key(row.toString(), "", "byte_str"), new Value("string".getBytes(UTF_8)));

    Scanner scanner = createMock(Scanner.class);
    expect(conn.createScanner(table, Authorizations.EMPTY)).andReturn(scanner);
    scanner.setRange(Range.exact(row));
    expect(scanner.iterator()).andReturn(entries.entrySet().iterator());

    replay(conn, scanner);

    Simple msgCopy = store.read(row, Simple.class);

    verify(conn, scanner);

    assertEquals(msg, msgCopy);
}

---

From source file:cereal.impl.StoreImplTest.java

License:Apache License

@Test
public void testEmptyRead() throws Exception {
    Simple msg = Simple.newBuilder().build();
    final Text row = new Text("read");

    Scanner scanner = createMock(Scanner.class);
    expect(conn.createScanner(table, Authorizations.EMPTY)).andReturn(scanner);
    scanner.setRange(Range.exact(row));/*from  w w  w .  j a  va2s.com*/
    expect(scanner.iterator()).andReturn(Collections.<Entry<Key, Value>>emptyIterator());

    replay(conn, scanner);

    Simple msgCopy = store.read(row, Simple.class);

    verify(conn, scanner);

    assertEquals(msg, msgCopy);
}

---

From source file:chaohBIM.ZipFileRecordReader.java

License:Apache License

/**
 * This is where the magic happens, each ZipEntry is decompressed and
 * readied for the Mapper. The contents of each file is held *in memory*
 * in a BytesWritable object./*from   w ww.jav  a 2 s .c  o m*/
 * 
 * If the ZipFileInputFormat has been set to Lenient (not the default),
 * certain exceptions will be gracefully ignored to prevent a larger job
 * from failing.
 */
@Override
public boolean nextKeyValue() throws IOException, InterruptedException {
    ZipEntry entry = null;
    try {
        entry = zip.getNextEntry();
    } catch (ZipException e) {
        if (ZipFileInputFormat.getLenient() == false)
            throw e;
    }

    // Sanity check
    if (entry == null) {
        isFinished = true;
        return false;
    }

    // Filename
    currentKey = new Text(zipfilename + "~" + entry.getName());

    // Read the file contents
    ByteArrayOutputStream bos = new ByteArrayOutputStream();
    byte[] temp = new byte[8192];//may be is too small for large file
    while (true) {
        int bytesRead = 0;
        try {
            bytesRead = zip.read(temp, 0, 8192);
        } catch (EOFException e) {
            if (ZipFileInputFormat.getLenient() == false)
                throw e;
            return false;
        }
        if (bytesRead > 0)
            bos.write(temp, 0, bytesRead);
        else
            break;
    }
    zip.closeEntry();

    // Uncompressed contents
    currentValue = new BytesWritable(bos.toByteArray());
    return true;
}

---

From source file:chaohParse.ZipFileRecordReader.java

License:Apache License

/**
 * This is where the magic happens, each ZipEntry is decompressed and
 * readied for the Mapper. The contents of each file is held *in memory*
 * in a BytesWritable object./*from  ww  w  .j av  a2 s .  c o  m*/
 * 
 * If the ZipFileInputFormat has been set to Lenient (not the default),
 * certain exceptions will be gracefully ignored to prevent a larger job
 * from failing.
 */
@Override
public boolean nextKeyValue() throws IOException, InterruptedException {
    ZipEntry entry = null;
    try {
        entry = zip.getNextEntry();
    } catch (ZipException e) {
        if (ZipFileInputFormat.getLenient() == false)
            throw e;
    }

    // Sanity check
    if (entry == null) {
        isFinished = true;
        return false;
    }

    // Filename
    currentKey = new Text(entry.getName());

    // Read the file contents
    ByteArrayOutputStream bos = new ByteArrayOutputStream();
    byte[] temp = new byte[8192];//may be is too small for large file
    while (true) {
        int bytesRead = 0;
        try {
            bytesRead = zip.read(temp, 0, 8192);
        } catch (EOFException e) {
            if (ZipFileInputFormat.getLenient() == false)
                throw e;
            return false;
        }
        if (bytesRead > 0)
            bos.write(temp, 0, bytesRead);
        else
            break;
    }
    zip.closeEntry();

    // Uncompressed contents
    currentValue = new BytesWritable(bos.toByteArray());
    return true;
}

---

From source file:cienciaCelularMR.FernetMapper.java

@Override
public void map(KeyMcell key, Text value, Context output) throws IOException, InterruptedException {
    try {/*from   w w w  .  j a v  a 2 s .c  o  m*/

        System.out.println("Entro al Map");
        System.out.println("Key del map fernet: " + key.toString());

        System.out.println("Fernet empezo a leer y guardar archivo .dat");
        try (FSDataInputStream fis = FileSystem.get(output.getConfiguration())
                .open(new Path(value.toString()))) {
            File archivo = new File("entradaFernet.dat");
            try (FileOutputStream fos = new FileOutputStream(archivo)) {
                byte[] buf = new byte[1024];
                int bytesRead;
                while ((bytesRead = fis.read(buf)) > 0) {
                    fos.write(buf, 0, bytesRead);
                    fos.flush();
                    output.progress();
                }
                fos.close();
                fis.close();
            }
        }

        System.out.println("Fernet termino de leer y guardar archivo .dat");
        Process process = new ProcessBuilder("fernet.exe", "--mode=" + key.getModoFernet().toString(),
                "--config=fernet.cfg", "entradaFernet.dat").start();

        InputStream is = process.getInputStream();
        InputStreamReader isr = new InputStreamReader(is);
        BufferedReader br = new BufferedReader(isr);
        String line;

        System.out.println("Fernet is running");
        System.out.println("La key de mapper fernet es: " + key);
        String res = "";
        while ((line = br.readLine()) != null) {
            res = res.concat(line);
            output.progress();
        }

        if ("point".equals(key.getModoFernet().toString())) {
            System.out.println("Fernet es point");
            String salidaName = "salidaFernet-" + key.getIdUsuario() + "." + key.getSubIdUsuario() + ".txt";
            FSDataOutputStream fs = FileSystem.get(output.getConfiguration()).create(new Path(salidaName));

            File salidaFile = new File("out_point.txt");
            if (salidaFile.exists()) {

                byte[] buffer = readFileToByteArray(new File("out_point.txt"));
                FernetOutput salida = new FernetOutput();
                salida.setFileName(new Text("out_point.txt"));
                salida.setSubId(key.getIdUsuario());
                salida.setValue(new BytesWritable(buffer));
                output.write(key, salida);
            }
        } else {
            File dir = new File(".");
            FileFilter fileFilter = new WildcardFileFilter("multi_*.txt");
            File[] files = dir.listFiles(fileFilter);
            for (File file : files) {
                byte[] buffer = readFileToByteArray(new File(file.getName()));
                FernetOutput salida = new FernetOutput();
                salida.setFileName(new Text(file.getName()));
                salida.setSubId(key.getIdUsuario());
                salida.setValue(new BytesWritable(buffer));
                output.write(key, salida);
            }
        }

    } catch (Exception ex) {
        String salidaName = "errorMapper-" + key.getIdUsuario() + "." + key.getSubIdUsuario() + ".txt";
        FSDataOutputStream fs = FileSystem.get(output.getConfiguration()).create(new Path(salidaName));
        fs.write(new Byte("Error en Mapper FERnet"));
        fs.write(new Byte("\n"));
        fs.flush();
        fs.close();

        Logger.getLogger(FernetMapper.class.getName()).log(Level.SEVERE, null, ex);
        throw ex;
    }
}

---

From source file:cienciaCelularMR.McellMapper.java

@Override
public void map(KeyMcell key, BytesWritable value, Context output) throws IOException, InterruptedException {
    try {//from  w  ww.  jav  a  2s. co m
        System.out.println("Entro al Map");
        System.out.println("Key del mcell mapper: " + key);

        byte[] arrayByte = value.copyBytes();
        File archivo = new File("entradaMap.mdl");
        try (FileOutputStream fos = new FileOutputStream(archivo)) {
            fos.write(arrayByte);
            fos.flush();
        }

        Process process = new ProcessBuilder("mcell.exe", "-errfile", "errorMcell.txt", "entradaMap.mdl")
                .start();

        InputStream is = process.getInputStream();
        InputStreamReader isr = new InputStreamReader(is);
        BufferedReader br = new BufferedReader(isr);
        String line;
        Matcher matcher;

        System.out.println("Mcell is running");
        String res = "";
        while ((line = br.readLine()) != null) {
            res = res.concat(line);
            res = res.concat("\n");
            output.progress();
            try {
                matcher = pattern.matcher(line);
                if (matcher.find()) {
                    int fieldCount;
                    Text[] fields;

                    fieldCount = matcher.groupCount();
                    fields = new Text[fieldCount];
                    for (int i = 0; i < fieldCount; i++) {
                        fields[i] = new Text(matcher.group(i + 1));
                    }
                    System.out.println("Progreso: " + Integer.parseInt(fields[0].toString()) + " de "
                            + Integer.parseInt(fields[1].toString()));
                }
            } catch (Exception ex) {
            }
        }

        File errorFile = new File("errorMcell.txt");
        if (errorFile.exists()) {
            InputStream in = new FileInputStream(errorFile);
            BufferedReader reader = new BufferedReader(new InputStreamReader(in));
            String l;
            while ((l = reader.readLine()) != null) {
                res = res.concat(l);
                res = res.concat("\n");
            }
        }

        mos.write("controloutput", key, new Text(res));

        //free memory
        res = "";
        System.out.println("Leyendo salida de MCell...");

        String salidaName = "salidaMCell-" + key.getIdUsuario() + "." + key.getSubIdUsuario() + ".dat";
        FSDataOutputStream fs = FileSystem.get(output.getConfiguration()).create(new Path(salidaName));

        File salidaFile = new File("joined_1.dat");

        if (salidaFile.exists()) {
            FileInputStream ios = new FileInputStream(salidaFile);
            byte[] buf = new byte[1024];
            int totalbytes = 0;
            int bytesRead;
            while ((bytesRead = ios.read(buf)) > 0) {
                totalbytes += bytesRead;
                fs.write(buf, 0, bytesRead);
                fs.flush();
                output.progress();
            }
            fs.close();
            ios.close();

            System.out.println("***Mcell termino de leer y guardar archivo .dat, tamao: " + totalbytes);
            System.out.println("Nombre que se le pasa a Fernet: " + salidaName);
            output.write(key, new Text(salidaName));
        } else {
            errorFile = new File("errorMcell.txt");
            if (errorFile.exists()) {
                InputStream in = new FileInputStream(errorFile);
                BufferedReader reader = new BufferedReader(new InputStreamReader(in));
                String l;
                while ((l = reader.readLine()) != null) {
                    res = res.concat(l);
                    res = res.concat("\n");
                }
                if (!"".equals(res)) {
                    mos.write("errormcell", key, new Text(res));
                }
            }
        }
    } catch (IOException | IllegalArgumentException | InterruptedException ex) {
        String salidaName = "errorMapper-" + key.getIdUsuario() + "." + key.getSubIdUsuario() + ".txt";
        FSDataOutputStream fs = FileSystem.get(output.getConfiguration()).create(new Path(salidaName));
        fs.write(new Byte("Error en Mapper MCell:"));
        fs.write(new Byte("\n"));
        fs.flush();
        fs.close();

        Logger.getLogger(McellMapper.class.getName()).log(Level.SEVERE, null, ex);
        throw new InterruptedException(ex.getMessage());
    }
}

---

From source file:cityhubtopten.Top10Mapper.java

public void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {

    String line = value.toString();

    String[] tokens = line.split(",");
    if (line.contains("Quality of Life")) {
        return;/*from   ww w.ja  va  2s. c o m*/
    }

    double quality = Double.parseDouble(tokens[6]);

    ToRecordMap.put(new QualityOfLife(quality), new Text(value));

    Iterator<Entry<QualityOfLife, Text>> iter = ToRecordMap.entrySet().iterator();

    Entry<QualityOfLife, Text> entry = null;

    while (ToRecordMap.size() > 10) {

        entry = iter.next();

        iter.remove();

    }

}

---

From source file:cityhubtopten.Top10Reducer.java

public void reduce(NullWritable key, Iterable<Text> values, Context context)
        throws IOException, InterruptedException {

    for (Text value : values) {

        String line = value.toString();

        if (line.length() > 0) {

            String[] tokens = line.split(",");

            //split the data and fetch salary

            double quality = Double.parseDouble(tokens[6]);

            //insert salary as key and entire row as value

            //tree map sort the records based on salary

            ToRecordMap.put(new QualityOfLife(quality), new Text(value));

        }//from   www  .j a v a  2s  .  c o m

    }

    // If we have more than ten records, remove the one with the lowest sal

    // As this tree map is sorted in descending order, the user with

    // the lowest sal is the last key.

    Iterator<Entry<QualityOfLife, Text>> iter = ToRecordMap.entrySet().iterator();

    Entry<QualityOfLife, Text> entry = null;

    while (ToRecordMap.size() > 10) {

        entry = iter.next();

        iter.remove();

    }

    for (Text t : ToRecordMap.descendingMap().values()) {

        // Output our ten records to the file system with a null key

        context.write(NullWritable.get(), t);

    }

}

---

From source file:cn.edu.hfut.dmic.webcollector.crawldb.Injector.java

public static void inject(Path crawlPath, CrawlDatums datums, Configuration conf)
        throws IOException, InterruptedException, ClassNotFoundException, Exception {
    Path crawldbPath = new Path(crawlPath, "crawldb");
    FileSystem fs = FileSystem.get(conf);
    Path tempdb = new Path(crawldbPath, "temp");
    if (fs.exists(tempdb)) {
        fs.delete(tempdb);//from   ww w  . j  ava  2  s  .  com
    }

    SequenceFile.Writer writer = new SequenceFile.Writer(fs, conf, new Path(tempdb, "info"), Text.class,
            CrawlDatum.class);

    for (CrawlDatum datum : datums) {

        String key = datum.getKey();
        writer.append(new Text(key), datum);
        LOG.info("inject:" + key);
    }
    writer.close();

    Path[] mergePaths = new Path[] { tempdb };

    Merge.merge(crawlPath, mergePaths, conf, "inject");
    Merge.install(crawlPath, conf);

    if (fs.exists(tempdb)) {
        fs.delete(tempdb);
    }

}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• 37
• 38
• 39
• 40
• 41
• 42
• »