Java tutorial
/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.arrow.vector.util; import java.io.DataInput; import java.io.IOException; import java.nio.ByteBuffer; import java.nio.CharBuffer; import java.nio.charset.CharacterCodingException; import java.nio.charset.Charset; import java.nio.charset.CharsetDecoder; import java.nio.charset.CharsetEncoder; import java.nio.charset.CodingErrorAction; import java.nio.charset.MalformedInputException; import java.text.CharacterIterator; import java.text.StringCharacterIterator; import java.util.Arrays; import com.fasterxml.jackson.core.JsonGenerationException; import com.fasterxml.jackson.core.JsonGenerator; import com.fasterxml.jackson.databind.SerializerProvider; import com.fasterxml.jackson.databind.annotation.JsonSerialize; import com.fasterxml.jackson.databind.ser.std.StdSerializer; /** * A simplified byte wrapper similar to Hadoop's Text class without all the dependencies. Lifted from Hadoop 2.7.1 */ @JsonSerialize(using = Text.TextSerializer.class) public class Text { private static ThreadLocal<CharsetEncoder> ENCODER_FACTORY = new ThreadLocal<CharsetEncoder>() { @Override protected CharsetEncoder initialValue() { return Charset.forName("UTF-8").newEncoder().onMalformedInput(CodingErrorAction.REPORT) .onUnmappableCharacter(CodingErrorAction.REPORT); } }; private static ThreadLocal<CharsetDecoder> DECODER_FACTORY = new ThreadLocal<CharsetDecoder>() { @Override protected CharsetDecoder initialValue() { return Charset.forName("UTF-8").newDecoder().onMalformedInput(CodingErrorAction.REPORT) .onUnmappableCharacter(CodingErrorAction.REPORT); } }; private static final byte[] EMPTY_BYTES = new byte[0]; private byte[] bytes; private int length; public Text() { bytes = EMPTY_BYTES; } /** * Construct from a string. */ public Text(String string) { set(string); } /** Construct from another text. */ public Text(Text utf8) { set(utf8); } /** * Construct from a byte array. */ public Text(byte[] utf8) { set(utf8); } /** * Get a copy of the bytes that is exactly the length of the data. See {@link #getBytes()} for faster access to the * underlying array. */ public byte[] copyBytes() { byte[] result = new byte[length]; System.arraycopy(bytes, 0, result, 0, length); return result; } /** * Returns the raw bytes; however, only data up to {@link #getLength()} is valid. Please use {@link #copyBytes()} if * you need the returned array to be precisely the length of the data. */ public byte[] getBytes() { return bytes; } /** Returns the number of bytes in the byte array */ public int getLength() { return length; } /** * Returns the Unicode Scalar Value (32-bit integer value) for the character at <code>position</code>. Note that this * method avoids using the converter or doing String instantiation * * @return the Unicode scalar value at position or -1 if the position is invalid or points to a trailing byte */ public int charAt(int position) { if (position > this.length) { return -1; // too long } if (position < 0) { return -1; // duh. } ByteBuffer bb = (ByteBuffer) ByteBuffer.wrap(bytes).position(position); return bytesToCodePoint(bb.slice()); } public int find(String what) { return find(what, 0); } /** * Finds any occurence of <code>what</code> in the backing buffer, starting as position <code>start</code>. The * starting position is measured in bytes and the return value is in terms of byte position in the buffer. The backing * buffer is not converted to a string for this operation. * * @return byte position of the first occurence of the search string in the UTF-8 buffer or -1 if not found */ public int find(String what, int start) { try { ByteBuffer src = ByteBuffer.wrap(this.bytes, 0, this.length); ByteBuffer tgt = encode(what); byte b = tgt.get(); src.position(start); while (src.hasRemaining()) { if (b == src.get()) { // matching first byte src.mark(); // save position in loop tgt.mark(); // save position in target boolean found = true; int pos = src.position() - 1; while (tgt.hasRemaining()) { if (!src.hasRemaining()) { // src expired first tgt.reset(); src.reset(); found = false; break; } if (!(tgt.get() == src.get())) { tgt.reset(); src.reset(); found = false; break; // no match } } if (found) { return pos; } } } return -1; // not found } catch (CharacterCodingException e) { // can't get here e.printStackTrace(); return -1; } } /** * Set to contain the contents of a string. */ public void set(String string) { try { ByteBuffer bb = encode(string, true); bytes = bb.array(); length = bb.limit(); } catch (CharacterCodingException e) { throw new RuntimeException("Should not have happened ", e); } } /** * Set to a utf8 byte array */ public void set(byte[] utf8) { set(utf8, 0, utf8.length); } /** copy a text. */ public void set(Text other) { set(other.getBytes(), 0, other.getLength()); } /** * Set the Text to range of bytes * * @param utf8 * the data to copy from * @param start * the first position of the new string * @param len * the number of bytes of the new string */ public void set(byte[] utf8, int start, int len) { setCapacity(len, false); System.arraycopy(utf8, start, bytes, 0, len); this.length = len; } /** * Append a range of bytes to the end of the given text * * @param utf8 * the data to copy from * @param start * the first position to append from utf8 * @param len * the number of bytes to append */ public void append(byte[] utf8, int start, int len) { setCapacity(length + len, true); System.arraycopy(utf8, start, bytes, length, len); length += len; } /** * Clear the string to empty. * * <em>Note</em>: For performance reasons, this call does not clear the underlying byte array that is retrievable via * {@link #getBytes()}. In order to free the byte-array memory, call {@link #set(byte[])} with an empty byte array * (For example, <code>new byte[0]</code>). */ public void clear() { length = 0; } /* * Sets the capacity of this Text object to <em>at least</em> <code>len</code> bytes. If the current buffer is longer, * then the capacity and existing content of the buffer are unchanged. If <code>len</code> is larger than the current * capacity, the Text object's capacity is increased to match. * * @param len the number of bytes we need * * @param keepData should the old data be kept */ private void setCapacity(int len, boolean keepData) { if (bytes == null || bytes.length < len) { if (bytes != null && keepData) { bytes = Arrays.copyOf(bytes, Math.max(len, length << 1)); } else { bytes = new byte[len]; } } } /** * Convert text back to string * * @see java.lang.Object#toString() */ @Override public String toString() { try { return decode(bytes, 0, length); } catch (CharacterCodingException e) { throw new RuntimeException("Should not have happened ", e); } } /** * Read a Text object whose length is already known. This allows creating Text from a stream which uses a different * serialization format. */ public void readWithKnownLength(DataInput in, int len) throws IOException { setCapacity(len, false); in.readFully(bytes, 0, len); length = len; } /** Returns true iff <code>o</code> is a Text with the same contents. */ @Override public boolean equals(Object o) { if (!(o instanceof Text)) { return false; } final Text that = (Text) o; if (this.getLength() != that.getLength()) { return false; } byte[] thisBytes = Arrays.copyOf(this.getBytes(), getLength()); byte[] thatBytes = Arrays.copyOf(that.getBytes(), getLength()); return Arrays.equals(thisBytes, thatBytes); } @Override public int hashCode() { return super.hashCode(); } // / STATIC UTILITIES FROM HERE DOWN /** * Converts the provided byte array to a String using the UTF-8 encoding. If the input is malformed, replace by a * default value. */ public static String decode(byte[] utf8) throws CharacterCodingException { return decode(ByteBuffer.wrap(utf8), true); } public static String decode(byte[] utf8, int start, int length) throws CharacterCodingException { return decode(ByteBuffer.wrap(utf8, start, length), true); } /** * Converts the provided byte array to a String using the UTF-8 encoding. If <code>replace</code> is true, then * malformed input is replaced with the substitution character, which is U+FFFD. Otherwise the method throws a * MalformedInputException. */ public static String decode(byte[] utf8, int start, int length, boolean replace) throws CharacterCodingException { return decode(ByteBuffer.wrap(utf8, start, length), replace); } private static String decode(ByteBuffer utf8, boolean replace) throws CharacterCodingException { CharsetDecoder decoder = DECODER_FACTORY.get(); if (replace) { decoder.onMalformedInput(java.nio.charset.CodingErrorAction.REPLACE); decoder.onUnmappableCharacter(CodingErrorAction.REPLACE); } String str = decoder.decode(utf8).toString(); // set decoder back to its default value: REPORT if (replace) { decoder.onMalformedInput(CodingErrorAction.REPORT); decoder.onUnmappableCharacter(CodingErrorAction.REPORT); } return str; } /** * Converts the provided String to bytes using the UTF-8 encoding. If the input is malformed, invalid chars are * replaced by a default value. * * @return ByteBuffer: bytes stores at ByteBuffer.array() and length is ByteBuffer.limit() */ public static ByteBuffer encode(String string) throws CharacterCodingException { return encode(string, true); } /** * Converts the provided String to bytes using the UTF-8 encoding. If <code>replace</code> is true, then malformed * input is replaced with the substitution character, which is U+FFFD. Otherwise the method throws a * MalformedInputException. * * @return ByteBuffer: bytes stores at ByteBuffer.array() and length is ByteBuffer.limit() */ public static ByteBuffer encode(String string, boolean replace) throws CharacterCodingException { CharsetEncoder encoder = ENCODER_FACTORY.get(); if (replace) { encoder.onMalformedInput(CodingErrorAction.REPLACE); encoder.onUnmappableCharacter(CodingErrorAction.REPLACE); } ByteBuffer bytes = encoder.encode(CharBuffer.wrap(string.toCharArray())); if (replace) { encoder.onMalformedInput(CodingErrorAction.REPORT); encoder.onUnmappableCharacter(CodingErrorAction.REPORT); } return bytes; } static final public int DEFAULT_MAX_LEN = 1024 * 1024; // //// states for validateUTF8 private static final int LEAD_BYTE = 0; private static final int TRAIL_BYTE_1 = 1; private static final int TRAIL_BYTE = 2; /** * Check if a byte array contains valid utf-8 * * @param utf8 * byte array * @throws MalformedInputException * if the byte array contains invalid utf-8 */ public static void validateUTF8(byte[] utf8) throws MalformedInputException { validateUTF8(utf8, 0, utf8.length); } /** * Check to see if a byte array is valid utf-8 * * @param utf8 * the array of bytes * @param start * the offset of the first byte in the array * @param len * the length of the byte sequence * @throws MalformedInputException * if the byte array contains invalid bytes */ public static void validateUTF8(byte[] utf8, int start, int len) throws MalformedInputException { int count = start; int leadByte = 0; int length = 0; int state = LEAD_BYTE; while (count < start + len) { int aByte = utf8[count] & 0xFF; switch (state) { case LEAD_BYTE: leadByte = aByte; length = bytesFromUTF8[aByte]; switch (length) { case 0: // check for ASCII if (leadByte > 0x7F) { throw new MalformedInputException(count); } break; case 1: if (leadByte < 0xC2 || leadByte > 0xDF) { throw new MalformedInputException(count); } state = TRAIL_BYTE_1; break; case 2: if (leadByte < 0xE0 || leadByte > 0xEF) { throw new MalformedInputException(count); } state = TRAIL_BYTE_1; break; case 3: if (leadByte < 0xF0 || leadByte > 0xF4) { throw new MalformedInputException(count); } state = TRAIL_BYTE_1; break; default: // too long! Longest valid UTF-8 is 4 bytes (lead + three) // or if < 0 we got a trail byte in the lead byte position throw new MalformedInputException(count); } // switch (length) break; case TRAIL_BYTE_1: if (leadByte == 0xF0 && aByte < 0x90) { throw new MalformedInputException(count); } if (leadByte == 0xF4 && aByte > 0x8F) { throw new MalformedInputException(count); } if (leadByte == 0xE0 && aByte < 0xA0) { throw new MalformedInputException(count); } if (leadByte == 0xED && aByte > 0x9F) { throw new MalformedInputException(count); } // falls through to regular trail-byte test!! case TRAIL_BYTE: if (aByte < 0x80 || aByte > 0xBF) { throw new MalformedInputException(count); } if (--length == 0) { state = LEAD_BYTE; } else { state = TRAIL_BYTE; } break; default: break; } // switch (state) count++; } } /** * Magic numbers for UTF-8. These are the number of bytes that <em>follow</em> a given lead byte. Trailing bytes have * the value -1. The values 4 and 5 are presented in this table, even though valid UTF-8 cannot include the five and * six byte sequences. */ static final int[] bytesFromUTF8 = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // trail bytes -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5 }; /** * Returns the next code point at the current position in the buffer. The buffer's position will be incremented. Any * mark set on this buffer will be changed by this method! */ public static int bytesToCodePoint(ByteBuffer bytes) { bytes.mark(); byte b = bytes.get(); bytes.reset(); int extraBytesToRead = bytesFromUTF8[(b & 0xFF)]; if (extraBytesToRead < 0) { return -1; // trailing byte! } int ch = 0; switch (extraBytesToRead) { case 5: ch += (bytes.get() & 0xFF); ch <<= 6; /* remember, illegal UTF-8 */ case 4: ch += (bytes.get() & 0xFF); ch <<= 6; /* remember, illegal UTF-8 */ case 3: ch += (bytes.get() & 0xFF); ch <<= 6; case 2: ch += (bytes.get() & 0xFF); ch <<= 6; case 1: ch += (bytes.get() & 0xFF); ch <<= 6; case 0: ch += (bytes.get() & 0xFF); } ch -= offsetsFromUTF8[extraBytesToRead]; return ch; } static final int offsetsFromUTF8[] = { 0x00000000, 0x00003080, 0x000E2080, 0x03C82080, 0xFA082080, 0x82082080 }; /** * For the given string, returns the number of UTF-8 bytes required to encode the string. * * @param string * text to encode * @return number of UTF-8 bytes required to encode */ public static int utf8Length(String string) { CharacterIterator iter = new StringCharacterIterator(string); char ch = iter.first(); int size = 0; while (ch != CharacterIterator.DONE) { if ((ch >= 0xD800) && (ch < 0xDC00)) { // surrogate pair? char trail = iter.next(); if ((trail > 0xDBFF) && (trail < 0xE000)) { // valid pair size += 4; } else { // invalid pair size += 3; iter.previous(); // rewind one } } else if (ch < 0x80) { size++; } else if (ch < 0x800) { size += 2; } else { // ch < 0x10000, that is, the largest char value size += 3; } ch = iter.next(); } return size; } public static class TextSerializer extends StdSerializer<Text> { public TextSerializer() { super(Text.class); } @Override public void serialize(Text text, JsonGenerator jsonGenerator, SerializerProvider serializerProvider) throws IOException, JsonGenerationException { jsonGenerator.writeString(text.toString()); } } }