Example usage for org.apache.hadoop.io ArrayWritable get

Introduction

In this page you can find the example usage for org.apache.hadoop.io ArrayWritable get.

Prototype

public Writable[] get() 

Source Link

Usage

From source file:edu.ub.ahstfg.io.WritableConverter.java

License:Open Source License

/**
 * Converts String ArrayWritable to String LinkedList.
 * @param input String ArrayWritable to convert.
 * @return Converted String LinkedList.//from  w  ww .j  a  va 2 s.  c  o m
 */
public static LinkedList<String> arrayWritable2LinkedListString(ArrayWritable input) {
    LinkedList<String> ret = new LinkedList<String>();
    Writable[] ws = input.get();
    Text t;
    for (Writable w : ws) {
        t = (Text) w;
        ret.add(t.toString());
    }
    return ret;
}

---

From source file:edu.ub.ahstfg.io.WritableConverter.java

License:Open Source License

/**
 * Converts Long ArrayWritable to long static array.
 * @param input Long ArrayWritable to convert.
 * @return Converted long static array.//from w  w  w  . ja  va 2s . c om
 */
public static short[] arrayWritable2ShortArray(ArrayWritable input) {
    Writable[] ws = input.get();
    short[] ret = new short[ws.length];
    int i = 0;
    IntWritable t;
    for (Writable w : ws) {
        t = (IntWritable) w;
        ret[i] = (short) t.get();
        i++;
    }
    return ret;
}

---

From source file:edu.ub.ahstfg.io.WritableConverter.java

License:Open Source License

/**
 * Converts String ArrayWritable to string static array.
 * @param input String ArrayWritable to convert.
 * @return Converted String static array.
 *///from   ww  w  .j ava2s .  c  om
public static String[] arrayWritable2StringArray(ArrayWritable input) {
    Writable[] ws = input.get();
    String[] ret = new String[ws.length];
    int i = 0;
    Text t;
    for (Writable w : ws) {
        t = (Text) w;
        ret[i] = t.toString();
        i++;
    }
    return ret;
}

---

From source file:edu.ub.ahstfg.kmeans.KmeansMapper.java

License:Open Source License

@Override
public void map(IntWritable key, ArrayWritable value, OutputCollector<IntWritable, DocumentDistance> output,
        Reporter reporter) throws IOException {

    this.output = output;
    this.reporter = reporter;

    String centroidsPath = job.get(ParamSet.OLD_CENTROIDS_PATH);
    int K = job.getInt(ParamSet.K, 0);
    if (K <= 0) {
        return;//w  w w  . j  a v  a2 s. c  o  m
    }

    int nKeywords = job.getInt(ParamSet.NUM_KEYWORDS, 0);
    //int nTerms    = job.getInt(ParamSet.NUM_TERMS   , 0);
    boolean haveKeywords = nKeywords > 0;
    float wKeywords = job.getFloat(ParamSet.WEIGHT_KEYWORDS, (float) 0.5);
    float wTerms = job.getFloat(ParamSet.WEIGHT_TERMS, (float) 0.5);

    centroids = new Centroids(K, DocumentCentroid.class);
    centroids.fromHDFS(centroidsPath);

    filledCentroids = new int[K];
    for (int i = 0; i < filledCentroids.length; i++) {
        filledCentroids[i] = 0;
    }

    Writable[] ws = value.get();
    DocumentDescriptor doc;
    DocumentCentroid centroid;
    double keywordDistance, termDistance, docDistance;
    double finalDistance;
    int finalCentroid;
    for (int iDoc = 0; iDoc < ws.length; iDoc++) {
        doc = (DocumentDescriptor) ws[iDoc];
        if (doc != null) {
            finalDistance = -1.0;
            finalCentroid = -1;
            for (int iCentroid = 0; iCentroid < K; iCentroid++) {
                centroid = (DocumentCentroid) centroids.get(iCentroid);
                termDistance = Metrics.euclideanDistance(doc.getTermFreq(), centroid.getTermVector());
                if (haveKeywords) {
                    keywordDistance = Metrics.euclideanDistance(doc.getKeyFreq(), centroid.getKeywordVector());
                } else {
                    keywordDistance = 0.0;
                }
                docDistance = wKeywords * keywordDistance + wTerms * termDistance;
                if (finalDistance < 0 || finalDistance > docDistance) {
                    finalDistance = docDistance;
                    finalCentroid = iCentroid;
                }
            }
            output.collect(new IntWritable(finalCentroid), new DocumentDistance(doc, finalDistance));
            fillCentroid(finalCentroid, 1);
        }
    }
    outVoidCentroids(output);
}

---

From source file:edu.umn.cs.spatialHadoop.operations.DistributedJoin.java

License:Open Source License

private static long selfJoinLocal(Path in, Path out, OperationsParams params) throws IOException {
    if (isOneShotReadMode) {
        // Ensure all objects are read in one shot
        params.setInt(SpatialSite.MaxBytesInOneRead, -1);
        params.setInt(SpatialSite.MaxShapesInOneRead, -1);
    } else {//from ww  w  .  j  a va2s  .c  o m
        params.setInt(SpatialSite.MaxBytesInOneRead, maxBytesInOneRead);
        params.setInt(SpatialSite.MaxShapesInOneRead, maxShapesInOneRead);
    }
    ShapeArrayInputFormat inputFormat = new ShapeArrayInputFormat();
    JobConf job = new JobConf(params);
    FileInputFormat.addInputPath(job, in);
    InputSplit[] splits = inputFormat.getSplits(job, 1);
    FileSystem outFs = out.getFileSystem(params);
    final PrintStream writer = new PrintStream(outFs.create(out));

    // Process all input files
    long resultSize = 0;
    for (InputSplit split : splits) {
        ShapeArrayRecordReader reader = new ShapeArrayRecordReader(job, (FileSplit) split);
        final Text temp = new Text();

        Rectangle key = reader.createKey();
        ArrayWritable value = reader.createValue();
        if (reader.next(key, value)) {
            Shape[] writables = (Shape[]) value.get();
            resultSize += SpatialAlgorithms.SelfJoin_planeSweep(writables, true,
                    new OutputCollector<Shape, Shape>() {
                        @Override
                        public void collect(Shape r, Shape s) throws IOException {
                            writer.print(r.toText(temp));
                            writer.print(",");
                            writer.println(s.toText(temp));
                        }
                    }, null);
            if (reader.next(key, value)) {
                throw new RuntimeException("Error! Not all values read in one shot.");
            }
        }

        reader.close();
    }
    writer.close();

    return resultSize;
}

---

From source file:edu.umn.cs.spatialHadoop.osm.OSMToKML.java

License:Open Source License

/**
 * @param args//from w  w w .  j  a v a  2  s  .  co m
 * @throws IOException 
 */
public static void main(String[] args) throws IOException {
    final OperationsParams params = new OperationsParams(new GenericOptionsParser(args), false);
    if (!params.checkInputOutput()) {
        System.err.println("Please specify input and output");
        System.exit(1);
    }
    params.setClass("shape", OSMPolygon.class, Shape.class);
    Path inputPath = params.getInputPath();
    FileSystem inFs = inputPath.getFileSystem(params);
    ShapeArrayRecordReader in = new ShapeArrayRecordReader(params,
            new FileSplit(inputPath, 0, inFs.getFileStatus(inputPath).getLen(), new String[0]));
    Path outPath = params.getOutputPath();
    FileSystem outFs = outPath.getFileSystem(params);
    PrintWriter out;
    ZipOutputStream zipOut = null;
    if (outPath.getName().toLowerCase().endsWith(".kmz")) {
        // Create a KMZ file
        FSDataOutputStream kmzOut = outFs.create(outPath);
        zipOut = new ZipOutputStream(kmzOut);
        zipOut.putNextEntry(new ZipEntry("osm.kml"));
        out = new PrintWriter(zipOut);
    } else {
        out = new PrintWriter(outFs.create(outPath));
    }
    out.println("<?xml version='1.0' encoding='UTF-8'?>");
    out.println("<kml xmlns='http://www.opengis.net/kml/2.2'>");
    out.println("<Document>");
    writeAllStyles(out);
    Rectangle key = in.createKey();
    ArrayWritable values = in.createValue();
    while (in.next(key, values)) {
        System.out.println("Read " + values.get().length);
        for (Shape shape : (Shape[]) values.get()) {
            if (shape instanceof OSMPolygon) {
                out.println(OSMtoKMLElement((OSMPolygon) shape));
            }
        }
        out.println();
    }
    out.println("</Document>");
    out.println("</kml>");
    in.close();
    if (zipOut != null) {
        // KMZ file
        out.flush();
        zipOut.closeEntry();
        zipOut.close();
    } else {
        // KML file
        out.close();
    }
}

---

From source file:hydrograph.engine.cascading.scheme.hive.parquet.WritableFactory.java

License:Apache License

public static Tuple getTuple(ArrayWritable arrayWritable) {
    Tuple tuple = new Tuple();
    for (Writable writable : arrayWritable.get()) {
        tuple.append(getFieldType(writable, tuple));
    }//from  w w  w  .  j a  va  2 s .  c  o m
    return tuple;
}

---

From source file:hydrograph.engine.hive.scheme.ParquetWritableUtilsTest.java

License:Apache License

@Test
public void itShouldGetBytesWritable() {
    try {//  www.j  ava 2 s.  com
        ArrayWritable writable = ParquetWritableUtils.createStruct(obj, (StructObjectInspector) io);
        BytesWritable bw = (BytesWritable) writable.get()[1];
        Assert.assertTrue(writable.get()[1] instanceof BytesWritable);
        Assert.assertEquals("hive", new String(bw.get()));
    } catch (Exception e) {
        throw new RuntimeException(e);
    }
}

---

From source file:hydrograph.engine.hive.scheme.ParquetWritableUtilsTest.java

License:Apache License

@Test
public void itShouldGetDecimalWritable() {
    try {/*from   w  ww.  j av  a2s. co  m*/
        ArrayWritable writable = ParquetWritableUtils.createStruct(obj, (StructObjectInspector) io);
        Assert.assertTrue(writable.get()[2] instanceof HiveDecimalWritable);
    } catch (Exception e) {
        throw new RuntimeException(e);
    }
}

---

From source file:hydrograph.engine.hive.scheme.ParquetWritableUtilsTest.java

License:Apache License

@Test
public void itShouldGetArrayWritable() {
    try {//ww w. ja v a 2 s .c  om
        ArrayWritable writable = ParquetWritableUtils.createStruct(obj, (StructObjectInspector) io);
        Assert.assertTrue(writable.get()[3] instanceof ArrayWritable);
    } catch (Exception e) {
        throw new RuntimeException(e);
    }
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• »