List of usage examples for org.apache.cassandra.serializers CollectionSerializer pack
public static ByteBuffer pack(Collection<ByteBuffer> buffers, int elements, ProtocolVersion version)
From source file:org.elassandra.cluster.InternalCassandraClusterService.java
License:Apache License
public void upsertDocument(final IndicesService indicesService, final IndexRequest request, final IndexMetaData indexMetaData, boolean updateOperation) throws Exception { final IndexService indexService = indicesService.indexServiceSafe(request.index()); final IndexShard indexShard = indexService.shardSafe(0); final SourceToParse sourceToParse = SourceToParse.source(SourceToParse.Origin.PRIMARY, request.source()) .type(request.type()).id(request.id()); if (request.routing() != null) sourceToParse.routing(request.routing()); if (request.parent() != null) sourceToParse.parent(request.parent()); if (request.timestamp() != null) sourceToParse.timestamp(request.timestamp()); if (request.ttl() != null) sourceToParse.ttl(request.ttl()); final String keyspaceName = indexService.indexSettings().get(IndexMetaData.SETTING_KEYSPACE, request.index());/*from w ww .j a v a 2s . co m*/ final String cfName = typeToCfName(request.type()); final Engine.IndexingOperation operation = indexShard.prepareIndexOnPrimary(sourceToParse, request.version(), request.versionType(), request.canHaveDuplicates()); final Mapping update = operation.parsedDoc().dynamicMappingsUpdate(); final boolean dynamicMappingEnable = indexService.indexSettings().getAsBoolean("index.mapper.dynamic", true); if (update != null && dynamicMappingEnable) { if (logger.isDebugEnabled()) logger.debug("Document source={} require a blocking mapping update of [{}]", request.sourceAsMap(), indexService.index().name()); // blocking Elasticsearch mapping update (required to update cassandra schema before inserting a row, this is the cost of dynamic mapping) blockingMappingUpdate(indexService, request.type(), update.toString()); } // get the docMapper after a potential mapping update final DocumentMapper docMapper = indexShard.mapperService().documentMapperWithAutoCreate(request.type()) .getDocumentMapper(); // insert document into cassandra keyspace=index, table = type final Map<String, Object> sourceMap = request.sourceAsMap(); final Map<String, ObjectMapper> objectMappers = docMapper.objectMappers(); final DocumentFieldMappers fieldMappers = docMapper.mappers(); Long timestamp = null; if (docMapper.timestampFieldMapper().enabled() && request.timestamp() != null) { timestamp = docMapper.timestampFieldMapper().fieldType().value(request.timestamp()); } if (logger.isTraceEnabled()) logger.trace( "Insert metadata.version={} index=[{}] table=[{}] id=[{}] source={} fieldMappers={} objectMappers={} consistency={} ttl={}", state().metaData().version(), indexService.index().name(), cfName, request.id(), sourceMap, Lists.newArrayList(fieldMappers.iterator()), objectMappers, request.consistencyLevel().toCassandraConsistencyLevel(), request.ttl()); final CFMetaData metadata = getCFMetaData(keyspaceName, cfName); String id = request.id(); Map<String, ByteBuffer> map = new HashMap<String, ByteBuffer>(); if (request.parent() != null) sourceMap.put(ParentFieldMapper.NAME, request.parent()); // normalize the _id and may find some column value in _id. // if the provided columns does not contains all the primary key columns, parse the _id to populate the columns in map. boolean buildId = true; ArrayNode array = jsonMapper.createArrayNode(); for (ColumnDefinition cd : Iterables.concat(metadata.partitionKeyColumns(), metadata.clusteringColumns())) { if (cd.name.toString().equals("_id")) { sourceMap.put("_id", request.id()); } Object value = sourceMap.get(cd.name.toString()); if (value != null) { addToJsonArray(cd.type, value, array); } else { buildId = false; parseElasticId(request.index(), cfName, request.id(), sourceMap); } } if (buildId) { id = writeValueAsString(array); } // workaround because ParentFieldMapper.value() and UidFieldMapper.value() create an Uid. if (sourceMap.get(ParentFieldMapper.NAME) != null && ((String) sourceMap.get(ParentFieldMapper.NAME)).indexOf(Uid.DELIMITER) < 0) { sourceMap.put(ParentFieldMapper.NAME, request.type() + Uid.DELIMITER + sourceMap.get(ParentFieldMapper.NAME)); } if (docMapper.sourceMapper().enabled()) { sourceMap.put(SourceFieldMapper.NAME, request.source()); } for (String field : sourceMap.keySet()) { FieldMapper fieldMapper = fieldMappers.getMapper(field); Mapper mapper = (fieldMapper != null) ? fieldMapper : objectMappers.get(field); ByteBuffer colName; if (mapper == null) { if (dynamicMappingEnable) throw new MapperException("Unmapped field [" + field + "]"); colName = ByteBufferUtil.bytes(field); } else { colName = mapper.cqlName(); // cached ByteBuffer column name. } final ColumnDefinition cd = metadata.getColumnDefinition(colName); if (cd != null) { // we got a CQL column. Object fieldValue = sourceMap.get(field); try { if (fieldValue == null) { if (cd.type.isCollection()) { switch (((CollectionType<?>) cd.type).kind) { case LIST: case SET: map.put(field, CollectionSerializer.pack(Collections.emptyList(), 0, Server.VERSION_3)); break; case MAP: break; } } else { map.put(field, null); } continue; } if (mapper != null && mapper.cqlCollection().equals(CqlCollection.SINGLETON) && (fieldValue instanceof Collection)) { throw new MapperParsingException( "field " + fieldMapper.name() + " should be a single value"); } // hack to store percolate query as a string while mapper is an object mapper. if (metadata.cfName.equals("_percolator") && field.equals("query")) { if (cd.type.isCollection()) { switch (((CollectionType<?>) cd.type).kind) { case LIST: if (((ListType) cd.type).getElementsType().asCQL3Type().equals(CQL3Type.Native.TEXT) && !(fieldValue instanceof String)) { // opaque list of objects serialized to JSON text fieldValue = Collections.singletonList(stringify(fieldValue)); } break; case SET: if (((SetType) cd.type).getElementsType().asCQL3Type().equals(CQL3Type.Native.TEXT) && !(fieldValue instanceof String)) { // opaque set of objects serialized to JSON text fieldValue = Collections.singleton(stringify(fieldValue)); } break; } } else { if (cd.type.asCQL3Type().equals(CQL3Type.Native.TEXT) && !(fieldValue instanceof String)) { // opaque singleton object serialized to JSON text fieldValue = stringify(fieldValue); } } } map.put(field, serializeType(request.index(), cfName, cd.type, field, fieldValue, mapper)); } catch (Exception e) { logger.error("[{}].[{}] failed to parse field {}={}", e, request.index(), cfName, field, fieldValue); throw e; } } } String query; ByteBuffer[] values; if (request.autoGeneratedId() || request.opType() == OpType.CREATE) { boolean checkUniqueId = Booleans.parseBoolean(request.checkUniqueId(), (request.autoGeneratedId()) ? false : true); values = new ByteBuffer[map.size()]; query = buildInsertQuery(keyspaceName, cfName, map, id, checkUniqueId, (request.ttl() != null) ? request.ttl().getSeconds() : null, // ttl timestamp, values, 0); final boolean applied = processConditional(request.consistencyLevel().toCassandraConsistencyLevel(), (checkUniqueId) ? ConsistencyLevel.LOCAL_SERIAL : null, query, values); if (!applied) throw new DocumentAlreadyExistsException(indexShard.shardId(), cfName, request.id()); } else { values = new ByteBuffer[metadata.partitionKeyColumns().size() + metadata.clusteringColumns().size()]; int i = 0; for (ColumnDefinition cd : metadata.partitionKeyColumns()) values[i++] = map.get(cd.name.toString()); for (ColumnDefinition cd : metadata.clusteringColumns()) values[i++] = map.get(cd.name.toString()); query = String.format((Locale) null, "DELETE FROM \"%s\".\"%s\" WHERE %s %s", keyspaceName, cfName, metadata.getCqlFragments().pkWhere, (timestamp != null) ? "USING TIMESTAMP " + Long.toString(timestamp * 1000 - 1) : ""); process(request.consistencyLevel().toCassandraConsistencyLevel(), query, values); values = new ByteBuffer[map.size()]; query = buildInsertQuery(keyspaceName, cfName, map, id, false, (request.ttl() != null) ? request.ttl().getSeconds() : null, timestamp, values, 0); process(request.consistencyLevel().toCassandraConsistencyLevel(), null, query, values); } }
From source file:org.elassandra.cluster.InternalCassandraClusterService.java
License:Apache License
/** * Serialize a cassandra typed object./*from w ww . j a v a 2 s . c o m*/ * List of list converted to List, see https://www.elastic.co/guide/en/elasticsearch/reference/current/array.html * @param ksName * @param cfName * @param type * @param name * @param value * @param mapper * @return * @throws SyntaxException * @throws ConfigurationException * @throws IOException * @throws JsonMappingException * @throws JsonGenerationException */ public static ByteBuffer serializeType(final String ksName, final String cfName, final AbstractType type, final String name, final Object value, final Mapper mapper) throws SyntaxException, ConfigurationException, JsonGenerationException, JsonMappingException, IOException { if (value == null) { return null; } if (type instanceof UserType) { UserType udt = (UserType) type; ByteBuffer[] components = new ByteBuffer[udt.size()]; int i = 0; if (GEO_POINT_TYPE.equals(ByteBufferUtil.string(udt.name))) { GeoPoint geoPoint = new GeoPoint(); if (value instanceof String) { // parse from string lat,lon (ex: "41.12,-71.34") or geohash (ex:"drm3btev3e86") geoPoint.resetFromString((String) value); } else { // parse from lat, lon fields as map Map<String, Object> mapValue = (Map<String, Object>) value; geoPoint.reset((Double) mapValue.get(GeoPointFieldMapper.Names.LAT), (Double) mapValue.get(GeoPointFieldMapper.Names.LON)); } components[i++] = serializeType(ksName, cfName, udt.fieldType(0), GeoPointFieldMapper.Names.LAT, geoPoint.lat(), null); components[i++] = serializeType(ksName, cfName, udt.fieldType(1), GeoPointFieldMapper.Names.LON, geoPoint.lon(), null); } else if (COMPLETION_TYPE.equals(ByteBufferUtil.string(udt.name))) { // input list<text>, output text, weight int, payload text Map<String, Object> mapValue = (Map<String, Object>) value; components[i++] = serializeType(ksName, cfName, udt.fieldType(0), Fields.CONTENT_FIELD_NAME_INPUT, mapValue.get(Fields.CONTENT_FIELD_NAME_INPUT), null); components[i++] = serializeType(ksName, cfName, udt.fieldType(1), Fields.CONTENT_FIELD_NAME_OUTPUT, mapValue.get(Fields.CONTENT_FIELD_NAME_OUTPUT), null); components[i++] = serializeType(ksName, cfName, udt.fieldType(2), Fields.CONTENT_FIELD_NAME_WEIGHT, new Long((Integer) mapValue.get(Fields.CONTENT_FIELD_NAME_WEIGHT)), null); components[i++] = serializeType(ksName, cfName, udt.fieldType(3), Fields.CONTENT_FIELD_NAME_PAYLOAD, stringify(mapValue.get(Fields.CONTENT_FIELD_NAME_PAYLOAD)), null); } else { Map<String, Object> mapValue = (Map<String, Object>) value; for (int j = 0; j < udt.size(); j++) { String subName = UTF8Type.instance.compose(udt.fieldName(j)); AbstractType<?> subType = udt.fieldType(j); Object subValue = mapValue.get(subName); Mapper subMapper = (mapper instanceof ObjectMapper) ? ((ObjectMapper) mapper).getMapper(subName) : null; components[i++] = serializeType(ksName, cfName, subType, subName, subValue, subMapper); } } return TupleType.buildValue(components); } else if (type instanceof MapType) { MapType mapType = InternalCassandraClusterService.getMapType(ksName, cfName, name); MapSerializer serializer = mapType.getSerializer(); Map map = (Map) value; List<ByteBuffer> buffers = serializer.serializeValues((Map) value); return CollectionSerializer.pack(buffers, map.size(), Server.VERSION_3); } else if (type instanceof CollectionType) { AbstractType elementType = (type instanceof ListType) ? ((ListType) type).getElementsType() : ((SetType) type).getElementsType(); if (elementType instanceof UserType && InternalCassandraClusterService.GEO_POINT_TYPE .equals(ByteBufferUtil.string(((UserType) elementType).name)) && value instanceof List && ((List) value).get(0) instanceof Double) { // geo_point as array of double lon,lat like [1.2, 1.3] UserType udt = (UserType) elementType; List<Double> values = (List<Double>) value; ByteBuffer[] elements = new ByteBuffer[] { serializeType(ksName, cfName, udt.fieldType(0), GeoPointFieldMapper.Names.LAT, values.get(1), null), serializeType(ksName, cfName, udt.fieldType(1), GeoPointFieldMapper.Names.LON, values.get(0), null) }; ByteBuffer geo_point = TupleType.buildValue(elements); return CollectionSerializer.pack(ImmutableList.of(geo_point), 1, Server.VERSION_3); } if (value instanceof Collection) { // list of elementType List<ByteBuffer> elements = new ArrayList<ByteBuffer>(); for (Object v : flattenCollection((Collection) value)) { ByteBuffer bb = serializeType(ksName, cfName, elementType, name, v, mapper); elements.add(bb); } return CollectionSerializer.pack(elements, elements.size(), Server.VERSION_3); } else { // singleton list ByteBuffer bb = serializeType(ksName, cfName, elementType, name, value, mapper); return CollectionSerializer.pack(ImmutableList.of(bb), 1, Server.VERSION_3); } } else { // Native cassandra type, encoded with mapper if available. if (mapper != null && mapper instanceof FieldMapper) { return type.decompose(value((FieldMapper) mapper, value)); } return type.decompose(value); } }