List of usage examples for com.fasterxml.jackson.core JsonToken START_ARRAY
JsonToken START_ARRAY
To view the source code for com.fasterxml.jackson.core JsonToken START_ARRAY.
Click Source Link
From source file:com.concentricsky.android.khanacademy.data.remote.LibraryUpdaterTask.java
private ContentValues parseObject(JsonParser parser, SQLiteDatabase tempDb, String parentId, int seq) throws JsonParseException, IOException { // TODO : Grab id of root topic here, and store it in shared prefs, in case it ever // changes. Currently we assume "root" and a change would be catastrophic. ContentValues result = new ContentValues(); ChildArrayResults childResults = null; boolean badKind = false; result.put("parentTopic_id", parentId); result.put("seq", seq); while (parser.nextValue() != JsonToken.END_OBJECT) { // Allows us to burn through the rest of the object once we discover it's an exercise or something else we don't care about. if (badKind) continue; String fieldName = parser.getCurrentName(); // Keys present will determine object type. if (stringFields.contains(fieldName)) { // Use getValueAsString over getText; getText returns "null" while getValueAsString returns null. String value = parser.getValueAsString(); result.put(fieldName, value); if ("id".equals(fieldName)) { if (childResults != null) { addParentIdToChildren(tempDb, childResults, value); }//from w w w.ja v a 2 s. com } } else if (intFields.contains(fieldName)) { result.put(fieldName, parser.getIntValue()); } else if (booleanFields.contains(fieldName)) { result.put(fieldName, parser.getBooleanValue()); } else if ("children".equals(fieldName)) { childResults = parseChildArray(parser, tempDb, result.containsKey("id") ? result.getAsString("id") : null); result.put("video_count", childResults.videoCount); result.put("child_kind", childResults.childKind); result.put("thumb_id", childResults.thumbId); } else if ("download_urls".equals(fieldName)) { parseDownloadUrls(parser, result); } else if (null == fieldName) { // Noop. Just in case. } else { JsonToken next = parser.getCurrentToken(); if (next == JsonToken.START_OBJECT || next == JsonToken.START_ARRAY) { // Skip this object or array, leaving us pointing at the matching end_object / end_array token. parser.skipChildren(); } } } // Ignore types we don't need. if (badKind) { return null; } // Having parsed this whole object, we can insert it. if (result.containsKey("kind")) { String kind = result.getAsString("kind"); if ("Topic".equals(kind)) { if (result.containsKey("id")) { result.put("_id", result.getAsString("id")); result.remove("id"); } if (result.containsKey("child_kind")) { String child_kind = result.getAsString("child_kind"); if ("Topic".equals(child_kind) || "Video".equals(child_kind)) { insertTopic(tempDb, result); } } } else if ("Video".equals(kind)) { if (result.containsKey("id")) { result.put("video_id", result.getAsString("id")); result.remove("id"); } insertTopicVideo(tempDb, result); insertVideo(tempDb, result); } } return result; }
From source file:bamboo.trove.common.BaseWarcDomainManager.java
private void parseJson(WarcProgressManager warc, InputStream in) throws IOException { Timer.Context ctx = bambooParseTimer.time(); JsonParser json = createParser(in);//from w w w . j a v a 2 s. c om JsonToken token = json.nextToken(); if (token == null) { ctx.stop(); throw new IllegalArgumentException("No JSON data found in response"); } if (!JsonToken.START_ARRAY.equals(token)) { ctx.stop(); throw new IllegalArgumentException("JSON response is not an array"); } try { long warcSize = 0; while (json.nextToken() == JsonToken.START_OBJECT) { Document d = objectMapper.readValue(json, Document.class); warcSize += d.getContentLength(); // Track it by batch IndexerDocument doc = warc.add(d); // Enqueue it for work filterQueue.offer(doc); } warcDocCountHistogram.update(warc.size()); warcSizeHistogram.update(warcSize); warc.setBatchBytes(warcSize); } finally { ctx.stop(); } }
From source file:io.pdef.json.JsonJacksonFormat.java
private List<?> readArray(final JsonParser parser) throws IOException { JsonToken current = parser.getCurrentToken(); if (current != JsonToken.START_ARRAY) { throw new JsonFormatException("Bad JSON string, failed to read an array"); }/* w w w .ja v a 2 s . c o m*/ List<Object> list = new ArrayList<Object>(); while (true) { JsonToken next = parser.nextToken(); if (next == null) { throw new JsonFormatException("End of file"); } else if (next == JsonToken.END_ARRAY) { break; } Object element = read(parser); list.add(element); } return list; }
From source file:bamboo.trove.full.FullReindexWarcManager.java
private LinkedList<ToIndex> getNextBatch() throws IOException { long startOfNextBatch = endOfBatchId + 1; URL url = new URL(bambooCollectionsUrl + "?start=" + startOfNextBatch + "&rows=" + bambooBatchSize); log.info("Contacting Bamboo for more IDs. start={}, rows={}", startOfNextBatch, bambooBatchSize); HttpURLConnection connection = (HttpURLConnection) url.openConnection(); InputStream in = new BufferedInputStream(connection.getInputStream()); ObjectMapper om = getObjectMapper(); JsonParser json = createParser(in);/* www . j a v a 2s . c o m*/ JsonToken token = json.nextToken(); if (token == null) { throw new IllegalArgumentException("No JSON data found in response"); } if (!JsonToken.START_ARRAY.equals(token)) { throw new IllegalArgumentException("JSON response is not an array"); } LinkedList<ToIndex> result = new LinkedList<>(); while (json.nextToken() == JsonToken.START_OBJECT) { result.add(new ToIndex(om.readValue(json, WarcToIndex.class))); } return result; }
From source file:com.cedarsoft.serialization.jackson.AbstractJacksonSerializer.java
@Nonnull protected <T> List<? extends T> deserializeArray(@Nonnull Class<T> type, @Nullable String propertyName, @Nonnull JsonParser deserializeFrom, @Nonnull Version formatVersion) throws IOException { JacksonParserWrapper parserWrapper = new JacksonParserWrapper(deserializeFrom); if (propertyName == null) { parserWrapper.verifyCurrentToken(JsonToken.START_ARRAY); } else {//from ww w. j a v a2s . co m parserWrapper.nextToken(); parserWrapper.verifyCurrentToken(JsonToken.FIELD_NAME); String currentName = parserWrapper.getCurrentName(); if (!propertyName.equals(currentName)) { throw new JsonParseException( "Invalid field. Expected <" + propertyName + "> but was <" + currentName + ">", parserWrapper.getCurrentLocation()); } parserWrapper.nextToken(); } List<T> deserialized = new ArrayList<T>(); while (deserializeFrom.nextToken() != JsonToken.END_ARRAY) { deserialized.add(deserialize(type, formatVersion, deserializeFrom)); } return deserialized; }
From source file:com.adobe.communities.ugc.migration.importer.GenericUGCImporter.java
/** * Handle each of the importable types of ugc content * @param jsonParser - the parsing stream * @param resource - the parent resource of whatever it is we're importing (must already exist) * @throws ServletException/*from w w w .j a v a 2s.c o m*/ * @throws IOException */ protected void importFile(final JsonParser jsonParser, final Resource resource, final ResourceResolver resolver) throws ServletException, IOException { final UGCImportHelper importHelper = new UGCImportHelper(); JsonToken token1 = jsonParser.getCurrentToken(); importHelper.setSocialUtils(socialUtils); if (token1.equals(JsonToken.START_OBJECT)) { jsonParser.nextToken(); if (jsonParser.getCurrentName().equals(ContentTypeDefinitions.LABEL_CONTENT_TYPE)) { jsonParser.nextToken(); final String contentType = jsonParser.getValueAsString(); if (contentType.equals(ContentTypeDefinitions.LABEL_QNA_FORUM)) { importHelper.setQnaForumOperations(qnaForumOperations); } else if (contentType.equals(ContentTypeDefinitions.LABEL_FORUM)) { importHelper.setForumOperations(forumOperations); } else if (contentType.equals(ContentTypeDefinitions.LABEL_CALENDAR)) { importHelper.setCalendarOperations(calendarOperations); } else if (contentType.equals(ContentTypeDefinitions.LABEL_JOURNAL)) { importHelper.setJournalOperations(journalOperations); } else if (contentType.equals(ContentTypeDefinitions.LABEL_FILELIBRARY)) { importHelper.setFileLibraryOperations(fileLibraryOperations); } importHelper.setTallyService(tallyOperationsService); // (everything potentially needs tally) importHelper.setCommentOperations(commentOperations); // nearly anything can have comments on it jsonParser.nextToken(); // content if (jsonParser.getCurrentName().equals(ContentTypeDefinitions.LABEL_CONTENT)) { jsonParser.nextToken(); token1 = jsonParser.getCurrentToken(); if (token1.equals(JsonToken.START_OBJECT) || token1.equals(JsonToken.START_ARRAY)) { if (!resolver.isLive()) { throw new ServletException("Resolver is already closed"); } } else { throw new ServletException("Start object token not found for content"); } if (token1.equals(JsonToken.START_OBJECT)) { try { if (contentType.equals(ContentTypeDefinitions.LABEL_QNA_FORUM)) { importHelper.importQnaContent(jsonParser, resource, resolver); } else if (contentType.equals(ContentTypeDefinitions.LABEL_FORUM)) { importHelper.importForumContent(jsonParser, resource, resolver); } else if (contentType.equals(ContentTypeDefinitions.LABEL_COMMENTS)) { importHelper.importCommentsContent(jsonParser, resource, resolver); } else if (contentType.equals(ContentTypeDefinitions.LABEL_JOURNAL)) { importHelper.importJournalContent(jsonParser, resource, resolver); } else if (contentType.equals(ContentTypeDefinitions.LABEL_FILELIBRARY)) { importHelper.importFileLibrary(jsonParser, resource, resolver); } else { LOG.info("Unsupported content type: {}", contentType); jsonParser.skipChildren(); } jsonParser.nextToken(); } catch (final IOException e) { throw new ServletException(e); } jsonParser.nextToken(); // skip over END_OBJECT } else { try { if (contentType.equals(ContentTypeDefinitions.LABEL_CALENDAR)) { importHelper.importCalendarContent(jsonParser, resource, resolver); } else if (contentType.equals(ContentTypeDefinitions.LABEL_TALLY)) { importHelper.importTallyContent(jsonParser, resource, resolver); } else { LOG.info("Unsupported content type: {}", contentType); jsonParser.skipChildren(); } jsonParser.nextToken(); } catch (final IOException e) { throw new ServletException(e); } jsonParser.nextToken(); // skip over END_ARRAY } } else { throw new ServletException("Content not found"); } } else { throw new ServletException("No content type specified"); } } else { throw new ServletException("Invalid Json format"); } }
From source file:org.seedstack.seed.core.internal.data.DataManagerImpl.java
private DataImporter<Object> consumeItems(JsonParser jsonParser, String group, String name, String acceptGroup, String acceptName) throws IOException { Map<String, DataImporterDefinition<Object>> dataImporterDefinitionMap = allDataImporters.get(group); if (dataImporterDefinitionMap == null) { throw SeedException.createNew(DataErrorCode.NO_IMPORTER_FOUND).put(GROUP, group).put(NAME, name); }//w w w .ja v a2s.c om DataImporterDefinition<Object> currentImporterDefinition = dataImporterDefinitionMap.get(name); if (currentImporterDefinition == null) { throw SeedException.createNew(DataErrorCode.NO_IMPORTER_FOUND).put(GROUP, group).put(NAME, name); } if (!group.equals(currentImporterDefinition.getGroup())) { throw SeedException.createNew(DataErrorCode.UNEXPECTED_DATA_TYPE) .put(DATA_SET, String.format(CLASSES_MAP_KEY, group, name)) .put(IMPORTER_CLASS, currentImporterDefinition.getDataImporterClass().getName()); } if (!name.equals(currentImporterDefinition.getName())) { throw SeedException.createNew(DataErrorCode.UNEXPECTED_DATA_TYPE) .put(DATA_SET, String.format(CLASSES_MAP_KEY, group, name)) .put(IMPORTER_CLASS, currentImporterDefinition.getDataImporterClass().getName()); } DataImporter<Object> currentDataImporter = null; if ((acceptGroup == null || acceptGroup.equals(group)) && (acceptName == null || acceptName.equals(name))) { currentDataImporter = injector.getInstance(currentImporterDefinition.getDataImporterClass()); // Check if items contains an array if (jsonParser.nextToken() != JsonToken.START_ARRAY) { throw new IllegalArgumentException("Items should be an array"); } jsonParser.nextToken(); // If the array is not empty consume it if (jsonParser.getCurrentToken() != JsonToken.END_ARRAY) { Iterator<Object> objectIterator = jsonParser .readValuesAs(currentImporterDefinition.getImportedClass()); while (objectIterator.hasNext()) { currentDataImporter.importData(objectIterator.next()); } // The array should end correctly if (jsonParser.getCurrentToken() != JsonToken.END_ARRAY) { throw new IllegalArgumentException("end array expected"); } } } // the data importer containing the data return currentDataImporter; }
From source file:com.concentricsky.android.khanacademy.data.remote.LibraryUpdaterTask.java
private ChildArrayResults parseChildArray(JsonParser parser, SQLiteDatabase tempDb, String parentTopic_id) throws JsonParseException, IOException { ChildArrayResults result = new ChildArrayResults(); int seq = 0;/*from w w w . ja v a2 s . c o m*/ JsonToken currentToken = parser.getCurrentToken(); if (currentToken == JsonToken.START_ARRAY) { while (parser.nextValue() == JsonToken.START_OBJECT) { // Otherwise, we will be at END_ARRAY here. ContentValues values = parseObject(parser, tempDb, parentTopic_id, seq++); if (values != null && values.containsKey("kind")) { String kind = values.getAsString("kind"); if ("Topic".equals(kind) && values.containsKey("_id")) { result.childKind = kind; result.childIds.add(values.getAsString("_id")); result.videoCount += values.getAsInteger("video_count"); if (result.thumbId == null && values.containsKey("thumb_id")) { // Return the first available thumb id as this topic's thumb id. result.thumbId = values.getAsString("thumb_id"); } } else if ("Video".equals(kind) && values.containsKey("readable_id")) { result.childKind = kind; result.childIds.add(values.getAsString("readable_id")); result.videoCount += 1; if (result.thumbId == null && values.containsKey("pngurl")) { // Return youtube_id of first video with a thumbnail as this topic's thumbnail id. result.thumbId = values.getAsString("youtube_id"); } } } } } return result; }
From source file:com.quinsoft.zeidon.standardoe.ActivateOisFromJsonStream.java
private void readEntity(String entityName) throws Exception { // Keeps track of whether the entity list starts with a [ or not. If there // is no [ then we are done reading entities of this type when we find the // end of the object. boolean entityArray = false; int twinCount = 0; JsonToken token = jp.getCurrentToken(); if (token == JsonToken.START_ARRAY) { token = jp.nextToken();/*w w w . j av a2s .c o m*/ entityArray = true; // Entity list started with [ } assert token == JsonToken.START_OBJECT; EntityDef entityDef = lodDef.getEntityDef(entityName, true, true); // Read tokens until we find the token that ends the current list of entities. while ((token = jp.nextToken()) != null) { twinCount++; if (token == JsonToken.END_ARRAY) break; if (token == JsonToken.END_OBJECT) { // If we get here then this should indicate an empty OI. Get the next // token, verify that it's an END_ARRAY, and return. token = jp.nextToken(); assert token == JsonToken.END_ARRAY; break; } // If there are multiple twins then the token is START_OBJECT to // indicate a new EI. if (token == JsonToken.START_OBJECT) { assert twinCount > 1; // Assert that we already created at least one EI. token = jp.nextToken(); } assert token == JsonToken.FIELD_NAME; EntityInstanceImpl ei = (EntityInstanceImpl) view.cursor(entityDef).createEntity(CursorPosition.LAST, CREATE_FLAGS); List<AttributeMeta> attributeMetas = new ArrayList<>(); // Read tokens until we find the token that ends the current entity. EntityMeta entityMeta = DEFAULT_ENTITY_META; while ((token = jp.nextToken()) != JsonToken.END_OBJECT) { String fieldName = jp.getCurrentName(); if (token == JsonToken.FIELD_NAME || token == JsonToken.START_OBJECT) token = jp.nextToken(); if (StringUtils.equals(fieldName, ".meta")) { entityMeta = readEntityMeta(ei); // Now that we have everything we can perform some processing. if (entityMeta.isLinkedSource) linkSources.put(entityMeta.entityKey, ei); else if (entityMeta.linkedSource != null) ei.linkInstances(linkSources.get(entityMeta.linkedSource)); continue; } if (fieldName.startsWith(".")) { AttributeMeta am = readAttributeMeta(ei, fieldName); attributeMetas.add(am); continue; } // Is this the start of an entity. if (token == JsonToken.START_ARRAY || token == JsonToken.START_OBJECT) { boolean recursiveChild = false; // Validate that the entity name is valid. EntityDef childEntity = lodDef.getEntityDef(fieldName, true, true); if (childEntity.getParent() != entityDef) { // Check to see the childEntity is a recursive child. if (entityDef.isRecursive()) { view.cursor(entityDef).setToSubobject(); recursiveChild = true; } else throw new ZeidonException("Parse error: %s is not a child of %s", fieldName, entityName); } readEntity(fieldName); if (recursiveChild) view.resetSubobject(); continue; } if (StringUtils.equals(jp.getText(), fieldName)) // If jp points to attr name, get next token. token = jp.nextToken(); // This better be an attribute // Try getting the attribute. We won't throw an exception (yet) if there // is no attribute with a matching name. AttributeDef attributeDef = entityDef.getAttribute(fieldName, false, true); if (attributeDef == null) { // We didn't find an attribute with a name matching fieldName. Do we allow // dynamic attributes for this entity? if (options.getAllowableDynamicEntities() == null || !options.getAllowableDynamicEntities().contains(entityDef.getName())) { entityDef.getAttribute(fieldName); // This will throw the exception. } // We are allowing dynamic attributes. Create one. DynamicAttributeDefConfiguration config = new DynamicAttributeDefConfiguration(); config.setAttributeName(fieldName); attributeDef = entityDef.createDynamicAttributeDef(config); } else if (attributeDef.isDerived()) // We'll ignore derived attributes. continue; Domain domain = attributeDef.getDomain(); Object internalValue = domain.convertExternalValue(task, ei.getAttribute(attributeDef), attributeDef, null, jp.getText()); ei.getAttribute(attributeDef).setInternalValue(internalValue, !attributeDef.isKey()); if (incremental) { // Since incremental flags are set, assume the attribute hasn't been // updated. We'll be told later if it has. AttributeValue attrib = ei.getInternalAttribute(attributeDef); attrib.setUpdated(false); } else { // If we just set the key then we'll assume the entity has // already been created. if (attributeDef.isKey()) ei.setIncrementalFlags(IncrementalEntityFlags.UPDATED); } } // while ( ( token = jp.nextToken() ) != JsonToken.END_OBJECT )... // Apply all the attribute metas to correctly set the attribute flags. for (AttributeMeta am : attributeMetas) am.apply(ei); // Now that we've updated everything, set the flags. if (incremental) { ei.setCreated(entityMeta.created); ei.setUpdated(entityMeta.updated); ei.setDeleted(entityMeta.deleted); ei.setIncluded(entityMeta.included); ei.setExcluded(entityMeta.excluded); if (entityMeta.incomplete) ei.setIncomplete(null); if (entityMeta.lazyLoaded != null) { String[] names = entityMeta.lazyLoaded.split(","); for (String name : names) ei.getEntitiesLoadedLazily().add(lodDef.getEntityDef(name, true, true)); } } // If the entity list didn't start with a [ then there is only one entity // in the list of twins so exit. if (entityArray == false) break; } // while ( ( token = jp.nextToken() ) != null )... }
From source file:eu.project.ttc.models.index.JsonTermIndexIO.java
/** * Loads the json-serialized term index into the param {@link TermIndex} object. * /*ww w. j a va 2s . c om*/ * @param reader * @param options * The deserialization {@link IOOptions}. * @return * @throws JsonParseException * @throws IOException */ public static TermIndex load(Reader reader, JsonOptions options) throws JsonParseException, IOException { TermIndex termIndex = null; JsonFactory jsonFactory = new JsonFactory(); JsonParser jp = jsonFactory.createParser(reader); // or Stream, Reader jp.enable(JsonParser.Feature.ALLOW_UNQUOTED_FIELD_NAMES); jp.enable(JsonParser.Feature.STRICT_DUPLICATE_DETECTION); String fieldname; String compLemma = null; int fileSource = -1; String wordLemma = null; String syntacticLabel = null; int begin = -1; int end = -1; int nbWordAnnos = -1; int nbSpottedTerms = -1; Term b; Term v; String text; String base; String variant; // String rule; String infoToken; String variantType; double variantScore; Map<Integer, String> inputSources = Maps.newTreeMap(); Map<Integer, List<TempVecEntry>> contextVectors = Maps.newHashMap(); OccurrenceStore occurrenceStore = null; // useful var for debug JsonToken tok; while ((tok = jp.nextToken()) != JsonToken.END_OBJECT) { fieldname = jp.getCurrentName(); if (METADATA.equals(fieldname)) { jp.nextToken(); String termIndexName = null; Lang lang = null; String corpusID = null; String occurrenceStorage = null; String occurrenceStoreURI = null; while ((tok = jp.nextToken()) != JsonToken.END_OBJECT) { fieldname = jp.getCurrentName(); if (LANG.equals(fieldname)) { lang = Lang.forName(jp.nextTextValue()); } else if (NAME.equals(fieldname)) { termIndexName = jp.nextTextValue(); } else if (NB_WORD_ANNOTATIONS.equals(fieldname)) { nbWordAnnos = jp.nextIntValue(-1); } else if (NB_SPOTTED_TERMS.equals(fieldname)) { nbSpottedTerms = jp.nextIntValue(-1); } else if (CORPUS_ID.equals(fieldname)) { corpusID = jp.nextTextValue(); } else if (OCCURRENCE_STORAGE.equals(fieldname)) { occurrenceStorage = jp.nextTextValue(); } else if (OCCURRENCE_MONGODB_STORE_URI.equals(fieldname)) { occurrenceStoreURI = jp.nextTextValue(); } } Preconditions.checkState(lang != null, "The property meta.lang must be defined"); Preconditions.checkState(termIndexName != null, "The property meta.name must be defined"); if (occurrenceStorage != null && occurrenceStorage.equals(OCCURRENCE_STORAGE_MONGODB)) { Preconditions.checkNotNull(occurrenceStoreURI, "Missing attribute " + OCCURRENCE_MONGODB_STORE_URI); occurrenceStore = new MongoDBOccurrenceStore(occurrenceStoreURI, OccurrenceStore.State.INDEXED); } else occurrenceStore = new MemoryOccurrenceStore(); termIndex = new MemoryTermIndex(termIndexName, lang, occurrenceStore); if (corpusID != null) termIndex.setCorpusId(corpusID); if (nbWordAnnos != -1) termIndex.setWordAnnotationsNum(nbWordAnnos); if (nbSpottedTerms != -1) termIndex.setSpottedTermsNum(nbSpottedTerms); if (options.isMetadataOnly()) return termIndex; } else if (WORDS.equals(fieldname)) { jp.nextToken(); while ((tok = jp.nextToken()) != JsonToken.END_ARRAY) { WordBuilder wordBuilder = WordBuilder.start(); while ((tok = jp.nextToken()) != JsonToken.END_OBJECT) { fieldname = jp.getCurrentName(); if (LEMMA.equals(fieldname)) wordBuilder.setLemma(jp.nextTextValue()); else if (COMPOUND_TYPE.equals(fieldname)) wordBuilder.setCompoundType(CompoundType.fromName(jp.nextTextValue())); else if (STEM.equals(fieldname)) wordBuilder.setStem(jp.nextTextValue()); else if (COMPONENTS.equals(fieldname)) { while ((tok = jp.nextToken()) != JsonToken.END_ARRAY) { while ((tok = jp.nextToken()) != JsonToken.END_OBJECT) { fieldname = jp.getCurrentName(); if (LEMMA.equals(fieldname)) compLemma = jp.nextTextValue(); else if (BEGIN.equals(fieldname)) begin = jp.nextIntValue(-2); else if (END.equals(fieldname)) end = jp.nextIntValue(-2); } wordBuilder.addComponent(begin, end, compLemma); } } } try { termIndex.addWord(wordBuilder.create()); } catch (Exception e) { LOGGER.error("Could not add word " + wordBuilder.getLemma() + " to term index", e); LOGGER.warn("Error ignored, trying ton continue the loading of TermIndex"); } } } else if (TERMS.equals(fieldname)) { jp.nextToken(); while ((tok = jp.nextToken()) != JsonToken.END_ARRAY) { TermBuilder builder = TermBuilder.start(termIndex); List<TempVecEntry> currentContextVector = Lists.newArrayList(); int currentTermId = -1; while ((tok = jp.nextToken()) != JsonToken.END_OBJECT) { fieldname = jp.getCurrentName(); if (GROUPING_KEY.equals(fieldname)) builder.setGroupingKey(jp.nextTextValue()); else if (SPOTTING_RULE.equals(fieldname)) builder.setSpottingRule(jp.nextTextValue()); else if (ID.equals(fieldname)) { currentTermId = jp.nextIntValue(-2); builder.setId(currentTermId); } else if (RANK.equals(fieldname)) { builder.setRank(jp.nextIntValue(-1)); } else if (FREQUENCY.equals(fieldname)) { builder.setFrequency(jp.nextIntValue(-1)); } else { if (FREQ_NORM.equals(fieldname)) { jp.nextToken(); builder.setFrequencyNorm((double) jp.getFloatValue()); } else if (SPECIFICITY.equals(fieldname)) { jp.nextToken(); builder.setSpecificity((double) jp.getDoubleValue()); } else if (GENERAL_FREQ_NORM.equals(fieldname)) { jp.nextToken(); builder.setGeneralFrequencyNorm((double) jp.getFloatValue()); } else if (WORDS.equals(fieldname)) { while ((tok = jp.nextToken()) != JsonToken.END_ARRAY) { wordLemma = null; syntacticLabel = null; while ((tok = jp.nextToken()) != JsonToken.END_OBJECT) { fieldname = jp.getCurrentName(); if (LEMMA.equals(fieldname)) wordLemma = jp.nextTextValue(); else if (SYN.equals(fieldname)) syntacticLabel = jp.nextTextValue(); } Preconditions.checkArgument(wordLemma != null, MSG_EXPECT_PROP_FOR_TERM_WORD, LEMMA); Preconditions.checkArgument(syntacticLabel != null, MSG_EXPECT_PROP_FOR_TERM_WORD, SYN); builder.addWord(termIndex.getWord(wordLemma), syntacticLabel); } // end words } else if (OCCURRENCES.equals(fieldname)) { tok = jp.nextToken(); if (tok == JsonToken.START_ARRAY) { while ((tok = jp.nextToken()) != JsonToken.END_ARRAY) { begin = -1; end = -1; fileSource = -1; text = null; while ((tok = jp.nextToken()) != JsonToken.END_OBJECT) { fieldname = jp.getCurrentName(); if (BEGIN.equals(fieldname)) begin = jp.nextIntValue(-1); else if (TEXT.equals(fieldname)) text = jp.nextTextValue(); else if (END.equals(fieldname)) end = jp.nextIntValue(-1); else if (FILE.equals(fieldname)) { fileSource = jp.nextIntValue(-1); } } Preconditions.checkArgument(begin != -1, MSG_EXPECT_PROP_FOR_OCCURRENCE, BEGIN); Preconditions.checkArgument(end != -1, MSG_EXPECT_PROP_FOR_OCCURRENCE, END); Preconditions.checkArgument(fileSource != -1, MSG_EXPECT_PROP_FOR_OCCURRENCE, FILE); Preconditions.checkNotNull(inputSources.get(fileSource), "No file source with id: %s", fileSource); Preconditions.checkNotNull(text, MSG_EXPECT_PROP_FOR_OCCURRENCE, TEXT); if (occurrenceStore.getStoreType() == OccurrenceStore.Type.MEMORY) builder.addOccurrence(begin, end, termIndex.getDocument(inputSources.get(fileSource)), text); } } // end occurrences } else if (CONTEXT.equals(fieldname)) { @SuppressWarnings("unused") int totalCooccs = 0; while ((tok = jp.nextToken()) != JsonToken.END_OBJECT) { fieldname = jp.getCurrentName(); if (TOTAL_COOCCURRENCES.equals(fieldname)) /* * value never used since the total will * be reincremented in the contextVector */ totalCooccs = jp.nextIntValue(-1); else if (CO_OCCURRENCES.equals(fieldname)) { jp.nextToken(); while ((tok = jp.nextToken()) != JsonToken.END_ARRAY) { TempVecEntry entry = new TempVecEntry(); while ((tok = jp.nextToken()) != JsonToken.END_OBJECT) { fieldname = jp.getCurrentName(); if (NB_COCCS.equals(fieldname)) entry.setNbCooccs(jp.nextIntValue(-1)); else if (ASSOC_RATE.equals(fieldname)) { jp.nextToken(); entry.setAssocRate(jp.getFloatValue()); } else if (CO_TERM.equals(fieldname)) entry.setTermGroupingKey(jp.nextTextValue()); else if (FILE.equals(fieldname)) { fileSource = jp.nextIntValue(-1); } } currentContextVector.add(entry); } } } } } //end if fieldname } // end term object try { builder.createAndAddToIndex(); } catch (Exception e) { LOGGER.error("Could not add term " + builder.getGroupingKey() + " to term index", e); LOGGER.warn("Error ignored, trying ton continue the loading of TermIndex"); } if (options.isWithContexts()) contextVectors.put(currentTermId, currentContextVector); } // end array of terms } else if (INPUT_SOURCES.equals(fieldname)) { jp.nextToken(); while ((tok = jp.nextToken()) != JsonToken.END_OBJECT) { String id = jp.getCurrentName(); try { inputSources.put(Integer.parseInt(id), jp.nextTextValue()); } catch (NumberFormatException e) { IOUtils.closeQuietly(jp); throw new IllegalArgumentException("Bad format for input source key: " + id); } } } else if (TERM_VARIATIONS.equals(fieldname)) { jp.nextToken(); while ((tok = jp.nextToken()) != JsonToken.END_ARRAY) { base = null; variant = null; infoToken = null; variantType = null; variantScore = 0; while ((tok = jp.nextToken()) != JsonToken.END_OBJECT) { fieldname = jp.getCurrentName(); if (BASE.equals(fieldname)) base = jp.nextTextValue(); else if (VARIANT.equals(fieldname)) variant = jp.nextTextValue(); else if (VARIANT_TYPE.equals(fieldname)) variantType = jp.nextTextValue(); else if (VARIANT_SCORE.equals(fieldname)) { jp.nextToken(); variantScore = jp.getDoubleValue(); } else if (INFO.equals(fieldname)) infoToken = jp.nextTextValue(); } // end syntactic variant object Preconditions.checkNotNull(base, MSG_EXPECT_PROP_FOR_VAR, BASE); Preconditions.checkNotNull(variant, MSG_EXPECT_PROP_FOR_VAR, VARIANT); Preconditions.checkNotNull(infoToken, MSG_EXPECT_PROP_FOR_VAR, INFO); b = termIndex.getTermByGroupingKey(base); v = termIndex.getTermByGroupingKey(variant); if (b != null && v != null) { VariationType vType = VariationType.fromShortName(variantType); TermVariation tv = new TermVariation(vType, b, v, vType == VariationType.GRAPHICAL ? Double.parseDouble(infoToken) : infoToken); tv.setScore(variantScore); b.addTermVariation(tv); } else { if (b == null) LOGGER.warn("Could not build variant because term \"{}\" was not found.", base); if (v == null) LOGGER.warn("Could not build variant because term \"{}\" was not found.", variant); } // Preconditions.checkNotNull(b, MSG_TERM_DOES_NOT_EXIST, base); // Preconditions.checkNotNull(v, MSG_TERM_DOES_NOT_EXIST, variant); } // end syntactic variations array } } jp.close(); if (options.isWithContexts()) { /* * map term ids with terms in context vectors and * set context vectors */ List<TempVecEntry> currentTempVecList; Term term = null; Term coTerm = null; ContextVector contextVector; for (int termId : contextVectors.keySet()) { currentTempVecList = contextVectors.get(termId); term = termIndex.getTermById(termId); contextVector = new ContextVector(term); for (TempVecEntry tempVecEntry : currentTempVecList) { coTerm = termIndex.getTermByGroupingKey(tempVecEntry.getTermGroupingKey()); contextVector.addEntry(coTerm, tempVecEntry.getNbCooccs(), tempVecEntry.getAssocRate()); } if (!contextVector.getEntries().isEmpty()) term.setContextVector(contextVector); } } return termIndex; }