List of usage examples for org.apache.lucene.util UnicodeUtil UTF8toUTF16
public static int UTF8toUTF16(byte[] utf8, int offset, int length, char[] out)
From source file:com.github.s4ke.moar.lucene.query.ByteCharSeq.java
License:Open Source License
@Override public int codePoint(int index) { //FIXME: is this the correct behaviour? this.tmpByte[0] = this.contents.bytes[index]; UnicodeUtil.UTF8toUTF16(this.tmpByte, 0, 1, this.tmpChar); return this.tmpChar[0] & 0xFFFF; }
From source file:fi.nationallibrary.ndl.solr.schema.CompressedField.java
License:Apache License
/** Decompress the byte array previously returned by * compressString back into a String/*from w ww . j a v a 2 s. com*/ * @param value the string encoded as a UTF-8 compressed byte array * @param offset the offset in the buffer that the string starts * @param length the length of the string */ public static String decompressString(byte[] value, int offset, int length) throws DataFormatException { UnicodeUtil.UTF16Result result = new UnicodeUtil.UTF16Result(); final byte[] bytes = decompress(value, offset, length); UnicodeUtil.UTF8toUTF16(bytes, 0, bytes.length, result); return new String(result.result, 0, result.length); }
From source file:io.crate.operation.scalar.string.LowerFunction.java
License:Apache License
@Override public BytesRef evaluate(Input<Object>... args) { Object stringValue = args[0].value(); if (stringValue == null) { return null; }/*from w w w.ja v a 2 s.c o m*/ BytesRef inputByteRef = BytesRefs.toBytesRef(stringValue); char[] ref = new char[inputByteRef.length]; int len = UnicodeUtil.UTF8toUTF16(inputByteRef.bytes, inputByteRef.offset, inputByteRef.length, ref); charUtils.toLowerCase(ref, 0, len); byte[] res = new byte[UnicodeUtil.MAX_UTF8_BYTES_PER_CHAR * len]; len = UnicodeUtil.UTF16toUTF8(ref, 0, len, res); return new BytesRef(res, 0, len); }
From source file:io.crate.operation.scalar.string.UpperFunction.java
License:Apache License
@Override public BytesRef evaluate(Input<Object>... args) { Object stringValue = args[0].value(); if (stringValue == null) { return null; }/* w w w . j a v a 2s. c om*/ BytesRef inputByteRef = BytesRefs.toBytesRef(stringValue); char[] ref = new char[inputByteRef.length]; int len = UnicodeUtil.UTF8toUTF16(inputByteRef.bytes, inputByteRef.offset, inputByteRef.length, ref); charUtils.toUpperCase(ref, 0, len); byte[] res = new byte[UnicodeUtil.MAX_UTF8_BYTES_PER_CHAR * len]; len = UnicodeUtil.UTF16toUTF8(ref, 0, len, res); return new BytesRef(res, 0, len); }
From source file:org.elasticsearch.action.search.type.TransportSearchHelper.java
License:Apache License
public static ParsedScrollId parseScrollId(String scrollId) { CharsRef spare = new CharsRef(); try {/*from w ww . ja v a 2s. c o m*/ byte[] decode = Base64.decode(scrollId, Base64.URL_SAFE); UnicodeUtil.UTF8toUTF16(decode, 0, decode.length, spare); } catch (IOException e) { throw new ElasticsearchIllegalArgumentException("Failed to decode scrollId", e); } String[] elements = Strings.splitStringToArray(spare, ';'); int index = 0; String type = elements[index++]; int contextSize = Integer.parseInt(elements[index++]); @SuppressWarnings({ "unchecked" }) Tuple<String, Long>[] context = new Tuple[contextSize]; for (int i = 0; i < contextSize; i++) { String element = elements[index++]; int sep = element.indexOf(':'); if (sep == -1) { throw new ElasticsearchIllegalArgumentException("Malformed scrollId [" + scrollId + "]"); } context[i] = new Tuple<String, Long>(element.substring(sep + 1), Long.parseLong(element.substring(0, sep))); } Map<String, String> attributes; int attributesSize = Integer.parseInt(elements[index++]); if (attributesSize == 0) { attributes = ImmutableMap.of(); } else { attributes = Maps.newHashMapWithExpectedSize(attributesSize); for (int i = 0; i < attributesSize; i++) { String element = elements[index++]; int sep = element.indexOf(':'); attributes.put(element.substring(0, sep), element.substring(sep + 1)); } } return new ParsedScrollId(scrollId, type, context, attributes); }
From source file:org.elasticsearch.memcached.netty.MemcachedDecoder.java
License:Apache License
@Override protected Object decode(ChannelHandlerContext ctx, Channel channel, ChannelBuffer buffer) throws Exception { MemcachedRestRequest request = this.request; if (request == null) { buffer.markReaderIndex();//from w ww . j a v a 2s. com if (buffer.readableBytes() < 1) { return null; } short magic = buffer.readUnsignedByte(); if (magic == 0x80) { if (buffer.readableBytes() < 23) { buffer.resetReaderIndex(); // but back magic return null; } short opcode = buffer.readUnsignedByte(); short keyLength = buffer.readShort(); short extraLength = buffer.readUnsignedByte(); short dataType = buffer.readUnsignedByte(); // unused short reserved = buffer.readShort(); // unused int totalBodyLength = buffer.readInt(); int opaque = buffer.readInt(); long cas = buffer.readLong(); // we want the whole of totalBodyLength; otherwise, keep waiting. if (buffer.readableBytes() < totalBodyLength) { buffer.resetReaderIndex(); return null; } buffer.skipBytes(extraLength); // get extras, can be empty char spare[] = new char[keyLength]; if (opcode == 0x00) { // GET byte[] key = new byte[keyLength]; buffer.readBytes(key); UnicodeUtil.UTF8toUTF16(key, 0, key.length, spare); request = new MemcachedRestRequest(RestRequest.Method.GET, new String(spare), key, -1, true); request.setOpaque(opaque); return request; } else if (opcode == 0x04) { // DELETE byte[] key = new byte[keyLength]; buffer.readBytes(key); UnicodeUtil.UTF8toUTF16(key, 0, key.length, spare); request = new MemcachedRestRequest(RestRequest.Method.DELETE, new String(spare), key, -1, true); request.setOpaque(opaque); return request; } else if (opcode == 0x01/* || opcode == 0x11*/) { // SET byte[] key = new byte[keyLength]; buffer.readBytes(key); UnicodeUtil.UTF8toUTF16(key, 0, key.length, spare); // the remainder of the message -- that is, totalLength - (keyLength + extraLength) should be the payload int size = totalBodyLength - keyLength - extraLength; request = new MemcachedRestRequest(RestRequest.Method.POST, new String(spare), key, size, true); request.setOpaque(opaque); request.setData(new ChannelBufferBytesReference(buffer.readSlice(size))); request.setQuiet(opcode == 0x11); return request; } else if (opcode == 0x0A || opcode == 0x10) { // NOOP or STATS // TODO once we support setQ we need to wait for them to flush ChannelBuffer writeBuffer = ChannelBuffers.dynamicBuffer(24); writeBuffer.writeByte(0x81); // magic writeBuffer.writeByte(opcode); // opcode writeBuffer.writeShort(0); // key length writeBuffer.writeByte(0); // extra length = flags + expiry writeBuffer.writeByte(0); // data type unused writeBuffer.writeShort(0x0000); // OK writeBuffer.writeInt(0); // data length writeBuffer.writeInt(opaque); // opaque writeBuffer.writeLong(0); // cas channel.write(writeBuffer); return MemcachedDispatcher.IGNORE_REQUEST; } else if (opcode == 0x07) { // QUIT channel.disconnect(); } else { logger.error("Unsupported opcode [0x{}], ignoring and closing connection", Integer.toHexString(opcode)); channel.disconnect(); return null; } } else { buffer.resetReaderIndex(); // reset to get to the first byte // need to read a header boolean done = false; StringBuffer sb = this.sb; int readableBytes = buffer.readableBytes(); for (int i = 0; i < readableBytes; i++) { byte next = buffer.readByte(); if (!ending && next == CR) { ending = true; } else if (ending && next == LF) { ending = false; done = true; break; } else if (ending) { logger.error("Corrupt stream, expected LF, found [0x{}]", Integer.toHexString(next)); throw new StreamCorruptedException("Expecting LF after CR"); } else { sb.append((char) next); } } if (!done) { // let's keep the buffer and bytes read // buffer.discardReadBytes(); buffer.markReaderIndex(); return null; } String[] args = lineSplit.split(sb); // we read the text, clear it sb.setLength(0); String cmd = args[0]; if ("get".equals(cmd)) { request = new MemcachedRestRequest(RestRequest.Method.GET, args[1], null, -1, false); if (args.length > 3) { BytesRef bytesRef = new BytesRef(args[2]); request.setData(new BytesArray(bytesRef)); } return request; } else if ("delete".equals(cmd)) { request = new MemcachedRestRequest(RestRequest.Method.DELETE, args[1], null, -1, false); // if (args.length > 3) { // request.setData(Unicode.fromStringAsBytes(args[2])); // } return request; } else if ("set".equals(cmd)) { this.request = new MemcachedRestRequest(RestRequest.Method.POST, args[1], null, Integer.parseInt(args[4]), false); buffer.markReaderIndex(); } else if ("version".equals(cmd)) { // sent as a noop byte[] bytes = Version.CURRENT.toString().getBytes(StandardCharsets.UTF_8); ChannelBuffer writeBuffer = ChannelBuffers.dynamicBuffer(bytes.length); writeBuffer.writeBytes(bytes); channel.write(writeBuffer); return MemcachedDispatcher.IGNORE_REQUEST; } else if ("quit".equals(cmd)) { if (channel.isConnected()) { // we maybe in the process of clearing the queued bits channel.disconnect(); } } else { logger.error("Unsupported command [{}], ignoring and closing connection", cmd); if (channel.isConnected()) { // we maybe in the process of clearing the queued bits channel.disconnect(); } return null; } } } else { if (buffer.readableBytes() < (request.getDataSize() + 2)) { return null; } BytesReference data = new ChannelBufferBytesReference(buffer.readSlice(request.getDataSize())); byte next = buffer.readByte(); if (next == CR) { next = buffer.readByte(); if (next == LF) { request.setData(data); // reset this.request = null; return request; } else { this.request = null; throw new StreamCorruptedException("Expecting separator after data block"); } } else { this.request = null; throw new StreamCorruptedException("Expecting separator after data block"); } } return null; }
From source file:org.elasticsearch.search.lookup.TermPosition.java
License:Apache License
public String payloadAsString() { if (payload != null && payload.length != 0) { UnicodeUtil.UTF8toUTF16(payload.bytes, payload.offset, payload.length, spare); return spare.toString(); } else {/* w w w .j av a2 s .c om*/ return null; } }
From source file:org.elasticsearch.xpack.core.security.authc.KeyAndTimestamp.java
License:Open Source License
KeyAndTimestamp(StreamInput input) throws IOException { timestamp = input.readVLong();/*from www .j a v a 2 s . c o m*/ byte[] keyBytes = input.readByteArray(); final char[] ref = new char[keyBytes.length]; int len = UnicodeUtil.UTF8toUTF16(keyBytes, 0, keyBytes.length, ref); key = new SecureString(Arrays.copyOfRange(ref, 0, len)); }
From source file:org.elasticsearch.xpack.security.authc.TokenService.java
License:Open Source License
private SecureString generateTokenKey() { byte[] keyBytes = new byte[KEY_BYTES]; byte[] encode = new byte[0]; char[] ref = new char[0]; try {/*w w w .ja va2s . com*/ secureRandom.nextBytes(keyBytes); encode = Base64.getUrlEncoder().withoutPadding().encode(keyBytes); ref = new char[encode.length]; int len = UnicodeUtil.UTF8toUTF16(encode, 0, encode.length, ref); return new SecureString(Arrays.copyOfRange(ref, 0, len)); } finally { Arrays.fill(keyBytes, (byte) 0x00); Arrays.fill(encode, (byte) 0x00); Arrays.fill(ref, (char) 0x00); } }
From source file:org.fastcatsearch.ir.document.CompressionTools.java
License:Apache License
/** Decompress the byte array previously returned by * compressString back into a String */ public static String decompressString(byte[] value, int offset, int length) throws DataFormatException { final byte[] bytes = decompress(value, offset, length); CharsRef result = new CharsRef(bytes.length); UnicodeUtil.UTF8toUTF16(bytes, 0, bytes.length, result); return new String(result.chars, 0, result.length); }