Example usage for com.google.gson.stream JsonReader JsonReader

Introduction

In this page you can find the example usage for com.google.gson.stream JsonReader JsonReader.

Prototype

public JsonReader(Reader in) 

Source Link

Document

Creates a new instance that reads a JSON-encoded stream from in .

Usage

From source file:org.citydb.citygml.validator.reader.cityjson.CityJSONValidator.java

License:Apache License

@Override
public void validate(InputFile inputFile) throws ValidationException {
    try {/*from   w  w w.j  a v a2  s. com*/
        inputStream = inputFile.openStream();

        // we do not really validate the file against the CityJSON schema
        // but only check whether Gson can parse it...
        try (JsonReader reader = new JsonReader(new InputStreamReader(inputStream))) {
            Gson gson = new GsonBuilder().registerTypeAdapterFactory(new CityJSONTypeAdapterFactory()).create();
            gson.fromJson(reader, CityJSON.class);
        }
    } catch (JsonParseException e) {
        if (!isAborted) {
            log.error("Invalid content: " + e.getMessage());
            hasErrors = true;
        }
    } catch (IOException e) {
        throw new ValidationException("Failed to validate CityJSON input file.", e);
    } finally {
        if (inputStream != null) {
            try {
                inputStream.close();
            } catch (IOException e) {
                //
            }
        }
    }

}

---

From source file:org.commoncrawl.mapred.ec2.parser.ParserMapper.java

License:Open Source License

private JsonObject parseResultToJsonObject(URL baseURL, ParseResult result, HTMLContent htmlMeta,
        Reporter reporter) throws IOException {

    JsonParser parser = new JsonParser();

    JsonObject objectOut = new JsonObject();

    objectOut.addProperty("type", "html-doc");

    safeSetString(objectOut, "title", result.getTitle());
    if (result.isFieldDirty(ParseResult.Field_TITLE))
        htmlMeta.setTitle(result.getTitle());

    if (result.getMetaTags().size() != 0) {
        JsonArray metaArray = new JsonArray();
        for (Meta meta : result.getMetaTags()) {
            JsonObject metaTag = new JsonObject();
            metaTag.addProperty("name", meta.getName());
            metaTag.addProperty("value", meta.getValue());

            metaArray.add(metaTag);/*  ww  w .j  a  va2s  .  c o  m*/
            htmlMeta.getMetaTags().add(new TextBytes(meta.getName().trim() + "\t" + meta.getValue()));
        }
        objectOut.add("meta_tags", metaArray);
    }
    if (result.getExtractedLinks().size() != 0) {
        JsonArray linkArray = new JsonArray();
        for (org.commoncrawl.service.parser.Link link : result.getExtractedLinks()) {
            try {
                JsonObject linkObj = parser.parse(new JsonReader(new StringReader(link.getAttributes())))
                        .getAsJsonObject();
                linkObj.addProperty("href", link.getUrl());
                linkArray.add(linkObj);

                HTMLLink linkMeta = new HTMLLink();
                linkMeta.setAttributes(link.getAttributes());
                linkMeta.setHref(link.getUrl());

                htmlMeta.getLinks().add(linkMeta);
            } catch (Exception e) {
                LOG.error("Error Parsing JSON Link Attributes for Link: " + link.getUrl() + " in Doc:" + baseURL
                        + " Exception:\n" + CCStringUtils.stringifyException(e));
                reporter.incrCounter(Counters.EXCEPTION_PARSING_LINK_JSON, 1);
            }
        }
        objectOut.add("links", linkArray);
    }
    return objectOut;
}

---

From source file:org.commoncrawl.mapred.ec2.parser.ParserMapper.java

License:Open Source License

/** 
 * some test code ... //from   w  ww  .  j a v  a 2s. c  om
 * 
 * @param args
 * @throws IOException
 */
public static void main(String[] args) throws IOException {
    Configuration conf = new Configuration();
    FileSystem fs = FileSystem.get(conf);
    Path pathToCrawlLog = new Path(args[0]);
    SequenceFile.Reader reader = new SequenceFile.Reader(fs, pathToCrawlLog, conf);

    Text url = new Text();
    CrawlURL urlData = new CrawlURL();

    ParserMapper mapper = new ParserMapper();
    MockReporter reporter = new MockReporter();
    final JsonParser parser = new JsonParser();

    while (reader.next(url, urlData)) {
        mapper.map(url, urlData, new OutputCollector<Text, ParseOutput>() {

            @Override
            public void collect(Text key, ParseOutput value) throws IOException {

                long timeStart = System.currentTimeMillis();
                JsonObject metadata = parser.parse(new JsonReader(new StringReader(value.getMetadata())))
                        .getAsJsonObject();
                long timeEnd = System.currentTimeMillis();

                if (metadata.has("parsed_as")) {
                    if (metadata.get("parsed_as").getAsString().equalsIgnoreCase("feed")) {
                        LOG.info("Got FEED for URL:" + key.toString() + " Parse Took:" + (timeEnd - timeStart)
                                + " Redirect:" + metadata.get("redirect"));
                        //LOG.info("FEED METADATA:" + metadata.toString());
                    }
                }

                //LOG.info("Key:" + key.toString() + " Metadata Size:" + value.getMetadataAsTextBytes().getLength());
                //LOG.info("Key:" + key.toString() + " Text-Size" + value.getTextContentAsTextBytes().getLength());
                //LOG.info("Key:" + key.toString() + " RAW-Size" + value.getRawContent().getCount());
            }
        }, reporter);
    }

    reader.close();
}

---

From source file:org.commoncrawl.service.parser.ec2.EC2ParserNode.java

License:Open Source License

private Pair<String, Long> checkoutFile() throws IOException {
    GenericUrl url = buildCheckoutURL();

    HttpRequest request = factory.buildGetRequest(url);

    HttpResponse response = request.execute();

    if (response.getStatusCode() == 200) {
        JsonParser parser = new JsonParser();
        JsonObject e = parser//from ww  w . j a v a  2 s .co m
                .parse(new JsonReader(new InputStreamReader(response.getContent(), Charset.forName("UTF-8"))))
                .getAsJsonObject();
        String logName = e.get("name").getAsString();
        long lastPos = e.get("lastPos").getAsLong();
        LOG.info("Got Name:" + logName + " Pos:" + lastPos);

        return new Pair<String, Long>(logName, lastPos);
    }
    return null;
}

---

From source file:org.commoncrawl.util.EC2MetadataTransferUtil.java

License:Open Source License

public static void main(String[] args) throws IOException {
    final String bucket = args[0];
    String paths = args[1];//from   ww  w  . j a  va2  s. c o  m

    System.out.println("Paths are:" + paths);
    JsonParser parser = new JsonParser();
    JsonReader reader = new JsonReader(new StringReader(paths));
    reader.setLenient(true);
    final JsonArray array = parser.parse(reader).getAsJsonArray();
    int pathCount = array.size();

    EC2MetadataTransferUtil util = new EC2MetadataTransferUtil(bucket, array);
}

---

From source file:org.commoncrawl.util.S3BulkTransferUtil.java

License:Open Source License

public static void main(String[] args) throws IOException {

    CommandLineParser parser = new GnuParser();

    try {//from   w ww  .  ja va 2s  .  c o m
        // parse the command line arguments
        CommandLine cmdLine = parser.parse(options, args);

        String s3AccessKey = cmdLine.getOptionValue("awsKey");
        String s3Secret = cmdLine.getOptionValue("awsSecret");
        String s3Bucket = cmdLine.getOptionValue("bucket");
        Path hdfsOutputPath = new Path(cmdLine.getOptionValue("outputPath"));
        JsonArray paths = new JsonArray();

        if (cmdLine.hasOption("path")) {
            String values[] = cmdLine.getOptionValues("path");
            for (String value : values) {
                paths.add(new JsonPrimitive(value));
            }
        }
        if (cmdLine.hasOption("paths")) {
            JsonParser jsonParser = new JsonParser();
            JsonReader reader = new JsonReader(new StringReader(cmdLine.getOptionValue("paths")));
            reader.setLenient(true);
            JsonArray array = jsonParser.parse(reader).getAsJsonArray();
            if (array != null) {
                paths.addAll(array);
            }
        }

        if (paths.size() == 0) {
            throw new IOException("No Input Paths Specified!");
        }

        LOG.info("Bucket:" + s3Bucket + " Target Paths:" + paths.toString());

        S3BulkTransferUtil util = new S3BulkTransferUtil(s3Bucket, s3AccessKey, s3Secret, paths,
                hdfsOutputPath);
    } catch (Exception e) {
        LOG.error(CCStringUtils.stringifyException(e));
        printUsage();
    }
}

---

From source file:org.couchbase.mock.subdoc.Executor.java

License:Apache License

private static <T> T parseStrictJson(String text, Class<T> klass) {
    try {//from   www .  jav  a  2s.c  om
        JSONValue.parseWithException(text);
    } catch (ParseException ex) {
        throw new JsonSyntaxException(ex);
    } catch (NumberFormatException ex2) {
        // Ignore number formats. GSON uses BigInteger if it's too big anyway. It's perfectly valid JSON
    }
    JsonReader reader = new JsonReader(new StringReader(text));
    reader.setLenient(false);
    return gs.fromJson(reader, klass);
}

---

From source file:org.cyanogenmod.changelog.ChangelogParser.java

License:Open Source License

public List<Change> readJsonStream(InputStream in) throws IOException {
    try (JsonReader reader = new JsonReader(new InputStreamReader(in, "UTF-8"))) {
        reader.setLenient(true); // strip XSSI protection
        return parseChangeInfoList(reader);
    }//from  w ww . j  ava 2s.co  m
}

---

From source file:org.dasd.ee.misc.Util.java

License:Open Source License

public static TestGroup loadSettings() {
    Gson gson = new Gson();
    File resources = new File("./resources/");
    if (!resources.exists()) {
        resources.mkdirs();/* w  ww .  j a  v  a  2s .co m*/
    }
    Map<String, File> resourceMap = Stream.of(resources.listFiles()).filter(x -> x.getName().contains(".json"))
            .collect(Collectors.toMap(x -> x.getName().substring(0, x.getName().lastIndexOf(".")), x -> x));
    File result = resourceMap
            .get(matchString("Settings File", resourceMap.keySet().toArray(new String[resourceMap.size()])));
    try {
        JsonReader reader = new JsonReader(new ProgressReader("Reading Settings", result));
        return gson.fromJson(reader, TestGroup.class);
    } catch (FileNotFoundException e) {
        return null;
    }
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• 37
• 38
• 39
• 40
• 41
• 42
• 43
• 44
• 45
• 46
• 47
• 48
• 49
• 50
• »