List of usage examples for io.vertx.core.json JsonObject getString
public String getString(String key)
From source file:org.entcore.feeder.dictionary.structures.DuplicateUsers.java
License:Open Source License
private void scoreDuplicates(final String profile, final JsonArray search, final Handler<Void> handler) { final String query = "START u=node:node_auto_index({luceneQuery}) " + "WHERE HEAD(u.profiles) = {profile} AND u.id <> {id} AND NOT(HAS(u.deleteDate)) " + "RETURN u.id as id, u.firstName as firstName, u.lastName as lastName, " + "u.birthDate as birthDate, u.email as email, u.source as source, u.disappearanceDate as disappearanceDate"; final JsonObject params = new JsonObject().put("profile", profile); TransactionHelper tx;/*from w w w .j av a2 s. c om*/ try { tx = TransactionManager.getTransaction(false); } catch (TransactionException e) { log.error("Error when find duplicate users.", e); return; } final JsonArray result = new fr.wseduc.webutils.collections.JsonArray(); for (int i = 0; i < search.size(); i++) { final JsonObject json = search.getJsonObject(i); final String firstNameAttr = luceneAttribute("firstName", json.getString("firstName"), 0.6); final String lastNameAttr = luceneAttribute("lastName", json.getString("lastName"), 0.6); String luceneQuery; if (firstNameAttr != null && lastNameAttr != null && !firstNameAttr.trim().isEmpty() && !lastNameAttr.trim().isEmpty()) { luceneQuery = firstNameAttr + " AND " + lastNameAttr; result.add(json); tx.add(query, params.copy().put("luceneQuery", luceneQuery).put("id", json.getString("id"))); } } tx.commit(new Handler<Message<JsonObject>>() { @Override public void handle(Message<JsonObject> event) { JsonArray results = event.body().getJsonArray("results"); if ("ok".equals(event.body().getString("status")) && results != null && results.size() > 0) { TransactionHelper tx; try { tx = TransactionManager.getTransaction(); tx.setAutoSend(false); } catch (TransactionException e) { log.error("Error when score duplicate users.", e); return; } for (int i = 0; i < results.size(); i++) { JsonArray findUsers = results.getJsonArray(i); if (findUsers == null || findUsers.size() == 0) continue; JsonObject searchUser = result.getJsonObject(i); calculateAndStoreScore(searchUser, findUsers, tx); } if (!tx.isEmpty()) { tx.commit(new Handler<Message<JsonObject>>() { @Override public void handle(Message<JsonObject> event) { if ("ok".equals(event.body().getString("status"))) { log.info("Mark duplicates " + profile + " finished."); } else { log.error("Error marking duplicates : " + event.body().getString("message")); } handler.handle(null); } }); } else { log.info("No duplicate user with score > 3 found in profile " + profile); handler.handle(null); } } else { if ("ok".equals(event.body().getString("status"))) { log.info("No duplicate user found in profile " + profile); } else { log.error( "Error finding users for search duplicates : " + event.body().getString("message")); } handler.handle(null); } } }); }
From source file:org.entcore.feeder.dictionary.structures.DuplicateUsers.java
License:Open Source License
private void calculateAndStoreScore(JsonObject searchUser, JsonArray findUsers, TransactionHelper tx) { String query = "MATCH (u:User {id : {sId}}), (d:User {id : {dId}}) " + "WHERE NOT({dId} IN coalesce(u.ignoreDuplicates, [])) AND NOT({sId} IN coalesce(d.ignoreDuplicates, [])) " + "AND (has(u.activationCode) OR has(d.activationCode)) " + "MERGE u-[:DUPLICATE {score:{score}}]-d "; JsonObject params = new JsonObject().put("sId", searchUser.getString("id")); final String lastName = cleanAttribute(searchUser.getString("lastName")); final String firstName = cleanAttribute(searchUser.getString("firstName")); final String birthDate = cleanAttribute(searchUser.getString("birthDate")); final String email = cleanAttribute(searchUser.getString("email")); final String source = searchUser.getString("source"); final Long disappearanceDate = searchUser.getLong("disappearanceDate"); for (int i = 0; i < findUsers.size(); i++) { int score = 2; JsonObject fu = findUsers.getJsonObject(i); score += exactMatch(lastName, cleanAttribute(fu.getString("lastName"))); score += exactMatch(firstName, cleanAttribute(fu.getString("firstName"))); score += exactMatch(birthDate, cleanAttribute(fu.getString("birthDate"))); score += exactMatch(email, cleanAttribute(fu.getString("email"))); if (score > 3 && ((!source.equals(fu.getString("source")) && (notDeduplicateSource.contains(source) ^ notDeduplicateSource.contains(fu.getString("source")))) || disappearanceDate != null || fu.getLong("disappearanceDate") != null)) { tx.add(query, params.copy().put("dId", fu.getString("id")).put("score", score)); }//from ww w. j a v a2 s . c o m } }
From source file:org.entcore.feeder.dictionary.structures.DuplicateUsers.java
License:Open Source License
public void autoMergeDuplicatesInStructure(final Handler<AsyncResult<JsonArray>> handler) { final Handler<JsonObject> duplicatesHandler = new Handler<JsonObject>() { @Override/*from w ww.j a v a2 s . c om*/ public void handle(JsonObject duplicatesRes) { JsonArray res = duplicatesRes.getJsonArray("result"); if ("ok".equals(duplicatesRes.getString("status")) && res != null && res.size() > 0) { try { final TransactionHelper tx = TransactionManager.getTransaction(); final AtomicInteger count = new AtomicInteger(res.size()); final Handler<JsonObject> mergeHandler = new Handler<JsonObject>() { @Override public void handle(JsonObject event) { decrementCount(count, tx); } }; for (Object o : res) { if (!(o instanceof JsonObject)) { decrementCount(count, tx); continue; } JsonObject j = (JsonObject) o; JsonObject u1 = j.getJsonObject("user1"); JsonObject u2 = j.getJsonObject("user2"); if (u1 != null && u2 != null) { mergeDuplicate(new ResultMessage(mergeHandler).put("userId1", u1.getString("id")) .put("userId2", u2.getString("id")), tx); log.info("AutoMerge duplicates - u1 : " + u1.encode() + ", u2 : " + u2.encode()); } else { decrementCount(count, tx); } } } catch (TransactionException e) { log.error("Error in automerge transaction", e); handler.handle(new DefaultAsyncResult<JsonArray>(e)); } } else { log.info("No duplicates automatically mergeable."); handler.handle(new DefaultAsyncResult<>(new fr.wseduc.webutils.collections.JsonArray())); } } private void decrementCount(AtomicInteger count, TransactionHelper tx) { if (count.decrementAndGet() == 0) { tx.commit(new Handler<Message<JsonObject>>() { @Override public void handle(Message<JsonObject> event) { if ("ok".equals(event.body().getString("status"))) { if (updateCourses) { AbstractTimetableImporter .updateMergedUsers(event.body().getJsonArray("results")); } handler.handle(new DefaultAsyncResult<>(event.body().getJsonArray("results"))); } else { log.error("Error in automerge duplicates transaction : " + event.body().getString("message")); handler.handle(new DefaultAsyncResult<JsonArray>( new TransactionException(event.body().getString("message")))); } } }); } } }; listDuplicates(new ResultMessage(duplicatesHandler).put("minScore", 5) .put("inSameStructure", autoMergeOnlyInSameStructure).put("inherit", false)); }
From source file:org.entcore.feeder.dictionary.structures.GraphData.java
License:Open Source License
static void loadData(final Neo4j neo4j, final Handler<Message<JsonObject>> handler) { String query = "MATCH (s:Structure) " + "OPTIONAL MATCH s<-[:DEPENDS]-(g:Group) " + "OPTIONAL MATCH s<-[:BELONGS]-(c:Class) " + "return s, collect(distinct g.externalId) as groups, collect(distinct c.externalId) as classes "; neo4j.execute(query, new JsonObject(), new Handler<Message<JsonObject>>() { @Override/*from w w w. j a va 2 s . c om*/ public void handle(Message<JsonObject> message) { String query = "MATCH (p:Profile) " + "OPTIONAL MATCH p<-[:COMPOSE]-(f:Function) " + "return p, collect(distinct f.externalId) as functions "; final AtomicInteger count = new AtomicInteger(2); neo4j.execute(query, new JsonObject(), new Handler<Message<JsonObject>>() { @Override public void handle(Message<JsonObject> message) { JsonArray res = message.body().getJsonArray("result"); if ("ok".equals(message.body().getString("status")) && res != null) { for (Object o : res) { if (!(o instanceof JsonObject)) continue; JsonObject r = (JsonObject) o; JsonObject p = r.getJsonObject("p", new JsonObject()).getJsonObject("data"); profiles.putIfAbsent(p.getString("externalId"), new Profile(p, r.getJsonArray("functions"))); } } if (handler != null && count.decrementAndGet() == 0) { handler.handle(message); } } }); JsonArray res = message.body().getJsonArray("result"); if ("ok".equals(message.body().getString("status")) && res != null) { for (Object o : res) { if (!(o instanceof JsonObject)) continue; JsonObject r = (JsonObject) o; JsonObject s = r.getJsonObject("s", new JsonObject()).getJsonObject("data"); Structure structure = new Structure(s, r.getJsonArray("groups"), r.getJsonArray("classes")); String externalId = s.getString("externalId"); structures.putIfAbsent(externalId, structure); String UAI = s.getString("UAI"); if (UAI != null && !UAI.trim().isEmpty()) { structuresByUAI.putIfAbsent(UAI, structure); } JsonArray joinKeys = s.getJsonArray("joinKey"); if (joinKeys != null && joinKeys.size() > 0) { for (Object key : joinKeys) { externalIdMapping.putIfAbsent(key.toString(), externalId); } } } } if (handler != null && count.decrementAndGet() == 0) { handler.handle(message); } } }); }
From source file:org.entcore.feeder.dictionary.structures.Group.java
License:Open Source License
public static void manualCreateOrUpdate(JsonObject object, String structureId, String classId, TransactionHelper transactionHelper) throws ValidationException { if (object == null) { throw new ValidationException("invalid.group"); } else {/*ww w. jav a2s . co m*/ final boolean create = (object.getString("id") == null || object.getString("id").trim().isEmpty()); final String id = create ? UUID.randomUUID().toString() : object.getString("id"); if (create) { object.put("id", id); } if (isNotEmpty(object.getString("name"))) { object.put("displayNameSearchField", Validator.sanitize(object.getString("name"))); } String query = "MERGE (t:Group:ManualGroup:Visible { id : {id}}) " + "SET " + Neo4jUtils.nodeSetPropertiesFromJson("t", object, "id") + "RETURN t.id as id "; transactionHelper.add(query, object); if (create) { if (structureId != null && !structureId.trim().isEmpty()) { String qs = "MATCH (s:Structure {id : {structureId}}), (g:Group {id : {groupId}}) " + "CREATE UNIQUE s<-[:DEPENDS]-g"; JsonObject ps = new JsonObject().put("groupId", id).put("structureId", structureId); transactionHelper.add(qs, ps); } if (classId != null && !classId.trim().isEmpty()) { String qs = "MATCH (s:Class {id : {classId}}), (g:Group {id : {groupId}}) " + "CREATE UNIQUE s<-[:DEPENDS]-g"; JsonObject ps = new JsonObject().put("groupId", id).put("classId", classId); transactionHelper.add(qs, ps); } } } }
From source file:org.entcore.feeder.dictionary.structures.Importer.java
License:Open Source License
private void loadFieldOfStudy(final Handler<Message<JsonObject>> handler) { Neo4j.getInstance().execute("MATCH (f:FieldOfStudy) return f.externalId as externalId, f.name as name", new JsonObject(), new Handler<Message<JsonObject>>() { @Override//from ww w . j a v a 2 s. c o m public void handle(Message<JsonObject> event) { final JsonArray res = event.body().getJsonArray("result"); if ("ok".equals(event.body().getString("status")) && res != null) { for (Object o : res) { if (!(o instanceof JsonObject)) continue; final JsonObject j = (JsonObject) o; fieldOfStudy.putIfAbsent(j.getString("name"), j.getString("externalId")); } } if (handler != null) { handler.handle(event); } } }); }
From source file:org.entcore.feeder.dictionary.structures.Importer.java
License:Open Source License
public Structure createOrUpdateStructure(JsonObject struct) { JsonArray groups = null;//from w w w .ja v a 2s . c o m if (struct != null) { groups = struct.getJsonArray("groups"); } final String error = structureValidator.validate(struct); Structure s = null; if (error != null) { report.addIgnored("Structure", error, struct); log.warn(error); } else { struct.put("source", currentSource); final String externalId = struct.getString("externalId"); if (groups != null) { for (Object gcMapping : groups) { if (!(gcMapping instanceof String)) continue; final String[] m = ((String) gcMapping).split("\\$"); final String groupCode = m[0]; if (groupCode == null || groupCode.isEmpty() || m.length < 3) continue; final List<String> classes = new LinkedList<>(); for (int i = 2; i < m.length; i++) { classes.add(externalId + "$" + m[i]); } if (!classes.isEmpty()) { groupClasses.put(externalId + "$" + groupCode, classes); } } } s = structures.get(externalId); if (s != null) { s.update(struct); } else { String UAI = struct.getString("UAI"); if (UAI != null) { s = structuresByUAI.get(UAI); } if (s != null) { structures.putIfAbsent(externalId, s); Object[] joinKeys = s.addJointure(externalId); if (joinKeys != null) { String origExternalId = s.getExternalId(); for (Object key : joinKeys) { externalIdMapping.putIfAbsent(key.toString(), origExternalId); } } } else { try { s = new Structure(externalId, struct); structures.putIfAbsent(externalId, s); s.create(); } catch (IllegalArgumentException e) { log.error(e.getMessage()); } } } } return s; }
From source file:org.entcore.feeder.dictionary.structures.Importer.java
License:Open Source License
public Profile createOrUpdateProfile(JsonObject profile) { final String error = profileValidator.validate(profile); Profile p = null;//from w ww .j a va 2 s . com if (error != null) { report.addIgnored("Profile", error, profile); log.warn(error); } else { String externalId = profile.getString("externalId"); p = profiles.get(externalId); if (p != null) { p.update(profile); } else { try { p = new Profile(externalId, profile); profiles.putIfAbsent(externalId, p); p.create(); } catch (IllegalArgumentException e) { log.error(e.getMessage()); } } } return p; }
From source file:org.entcore.feeder.dictionary.structures.Importer.java
License:Open Source License
public void createOrUpdateUser(JsonObject object, JsonArray linkStudent, boolean linkRelativeWithoutChild) { final String error = userValidator.validate(object); if (error != null) { report.addIgnored("Relative", error, object); log.warn(error);//from ww w. ja va2 s . c o m } else { object.put("source", currentSource); userImportedExternalId.add(object.getString("externalId")); String query = "MERGE (u:User { externalId : {externalId}}) " + "ON CREATE SET u.id = {id}, u.login = {login}, u.activationCode = {activationCode}, " + "u.displayName = {displayName}, u.created = {created} " + "WITH u " + "WHERE u.checksum IS NULL OR u.checksum <> {checksum} " + "SET " + Neo4jUtils.nodeSetPropertiesFromJson("u", object, "id", "externalId", "login", "activationCode", "displayName", "email", "created"); transactionHelper.add(query, object); checkUpdateEmail(object); if (linkStudent != null && linkStudent.size() > 0) { String query2 = "START u0=node:node_auto_index(externalId={externalId}), " + "s=node:node_auto_index({studentExternalIds}) " + "MATCH u0-[:MERGED*0..1]->u " + "WHERE NOT(HAS(u.mergedWith)) " + "MERGE u<-[:RELATED]-s "; JsonObject p = new JsonObject().put("externalId", object.getString("externalId")) .put("studentExternalIds", "externalId:" + Joiner.on(" OR externalId:").join(linkStudent)); transactionHelper.add(query2, p); } else if (linkRelativeWithoutChild) { final String externalId = object.getString("externalId"); JsonArray structures = getMappingStructures(object.getJsonArray("structures")); if (externalId != null && structures != null && structures.size() > 0) { JsonObject p = new JsonObject().put("userExternalId", externalId); String q1 = "MATCH (s:Structure)<-[:DEPENDS]-(g:ProfileGroup)-[:HAS_PROFILE]->(p:Profile), " + "(:User { externalId : {userExternalId}})-[:MERGED*0..1]->(u:User) " + "USING INDEX s:Structure(externalId) " + "USING INDEX p:Profile(externalId) " + "WHERE s.externalId IN {structuresAdmin} " + "AND p.externalId = {profileExternalId} AND NOT(HAS(u.mergedWith)) " + "MERGE u-[:IN]->g"; p.put("structuresAdmin", structures).put("profileExternalId", DefaultProfiles.RELATIVE_PROFILE_EXTERNAL_ID); transactionHelper.add(q1, p); } } } }
From source file:org.entcore.feeder.dictionary.structures.Importer.java
License:Open Source License
public void createOrUpdateGuest(JsonObject object, String[][] linkClasses) { final String error = userValidator.validate(object); if (error != null) { report.addIgnored("Guest", error, object); log.warn(error);/*ww w. j a va2 s.co m*/ } else { object.put("source", currentSource); final String externalId = object.getString("externalId"); userImportedExternalId.add(externalId); String query = "MERGE (u:User { externalId : {externalId}}) " + "ON CREATE SET u.id = {id}, u.login = {login}, u.activationCode = {activationCode}, " + "u.displayName = {displayName}, u.created = {created} " + "WITH u " + "WHERE u.checksum IS NULL OR u.checksum <> {checksum} " + "SET " + Neo4jUtils.nodeSetPropertiesFromJson("u", object, "id", "externalId", "login", "activationCode", "displayName", "email", "created"); transactionHelper.add(query, object); checkUpdateEmail(object); JsonArray structures = getMappingStructures(object.getJsonArray("structures")); if (externalId != null && structures != null && structures.size() > 0) { JsonObject p = new JsonObject().put("userExternalId", externalId); String q1 = "MATCH (s:Structure)<-[:DEPENDS]-(g:ProfileGroup)-[:HAS_PROFILE]->(p:Profile), " + "(:User { externalId : {userExternalId}})-[:MERGED*0..1]->(u:User) " + "USING INDEX s:Structure(externalId) " + "USING INDEX p:Profile(externalId) " + "WHERE s.externalId IN {structuresAdmin} " + "AND p.externalId = {profileExternalId} AND NOT(HAS(u.mergedWith)) " + "MERGE u-[:IN]->g"; p.put("structuresAdmin", structures).put("profileExternalId", DefaultProfiles.GUEST_PROFILE_EXTERNAL_ID); transactionHelper.add(q1, p); String qs = "MATCH (:User {externalId : {userExternalId}})-[r:IN|COMMUNIQUE]-(:Group)-[:DEPENDS]->(s:Structure) " + "WHERE NOT(s.externalId IN {structures}) AND (NOT(HAS(r.source)) OR r.source = {source}) " + "DELETE r"; JsonObject ps = new JsonObject().put("userExternalId", externalId).put("source", currentSource) .put("structures", structures); transactionHelper.add(qs, ps); } if (externalId != null && linkClasses != null) { JsonArray classes = new fr.wseduc.webutils.collections.JsonArray(); for (String[] structClass : linkClasses) { if (structClass != null && structClass[0] != null && structClass[1] != null) { String q = "MATCH (s:Structure)<-[:BELONGS]-(c:Class)<-[:DEPENDS]-(g:ProfileGroup)" + "-[:DEPENDS]->(pg:ProfileGroup)-[:HAS_PROFILE]->(p:Profile), " + "(:User { externalId : {userExternalId}})-[:MERGED*0..1]->(u:User) " + "USING INDEX s:Structure(externalId) " + "USING INDEX p:Profile(externalId) " + "WHERE s.externalId = {structure} AND c.externalId = {class} " + "AND p.externalId = {profileExternalId} AND NOT(HAS(u.mergedWith)) " + "MERGE u-[:IN]->g"; JsonObject p = new JsonObject().put("userExternalId", externalId) .put("profileExternalId", DefaultProfiles.GUEST_PROFILE_EXTERNAL_ID) .put("structure", structClass[0]).put("class", structClass[1]); transactionHelper.add(q, p); classes.add(structClass[1]); } } String q = "MATCH (:User {externalId : {userExternalId}})-[r:IN|COMMUNIQUE]-(:Group)-[:DEPENDS]->(c:Class) " + "WHERE NOT(c.externalId IN {classes}) AND (NOT(HAS(r.source)) OR r.source = {source}) " + "DELETE r"; JsonObject p = new JsonObject().put("userExternalId", externalId).put("source", currentSource) .put("classes", classes); transactionHelper.add(q, p); } } }