Example usage for org.apache.hadoop.io Text Text

List of usage examples for org.apache.hadoop.io Text Text

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.io.*
  5. Text
  6. Text

Introduction

In this page you can find the example usage for org.apache.hadoop.io Text Text.

Prototype

public Text(byte[] utf8)

Source Link

Document

Construct from a byte array.

Usage

From source file:co.cask.cdap.hive.objectinspector.SimpleMapEqualComparerTest.java

License:Apache License

@Test
public void testCompatibleType() throws SerDeException, IOException {
    // empty maps
    TextStringMapHolder o1 = new TextStringMapHolder();
    StructObjectInspector oi1 = (StructObjectInspector) ObjectInspectorFactory
            .getReflectionObjectInspector(TextStringMapHolder.class);

    LazySimpleSerDe serde = new LazySimpleSerDe();
    Configuration conf = new Configuration();
    Properties tbl = new Properties();
    tbl.setProperty(serdeConstants.LIST_COLUMNS, ObjectInspectorUtils.getFieldNames(oi1));
    tbl.setProperty(serdeConstants.LIST_COLUMN_TYPES, ObjectInspectorUtils.getFieldTypes(oi1));
    SerDeParameters serdeParams = LazySimpleSerDe.initSerdeParams(conf, tbl, LazySimpleSerDe.class.getName());
    serde.initialize(conf, tbl);//from  ww  w . j av a  2s .c  o m
    ObjectInspector oi2 = serde.getObjectInspector();

    Object o2 = serializeAndDeserialize(o1, oi1, serde, serdeParams);

    int rc = ObjectInspectorUtils.compare(o1, oi1, o2, oi2, new SimpleMapEqualComparer());
    Assert.assertEquals(0, rc);

    // equal maps
    o1.mMap.put(new Text("42"), "The answer to Life, Universe And Everything");
    o1.mMap.put(new Text("1729"), "A taxi cab number");
    o2 = serializeAndDeserialize(o1, oi1, serde, serdeParams);
    rc = ObjectInspectorUtils.compare(o1, oi1, o2, oi2, new SimpleMapEqualComparer());
    Assert.assertEquals(0, rc);

    // unequal maps
    o1.mMap.put(new Text("1729"), "Hardy-Ramanujan Number");
    rc = ObjectInspectorUtils.compare(o1, oi1, o2, oi2, new SimpleMapEqualComparer());
    Assert.assertFalse(0 == rc);
}

From source file:co.cask.cdap.hive.objectinspector.SimpleMapEqualComparerTest.java

License:Apache License

Object serializeAndDeserialize(StringTextMapHolder o1, StructObjectInspector oi1, LazySimpleSerDe serde,
        SerDeParameters serdeParams) throws IOException, SerDeException {
    ByteStream.Output serializeStream = new ByteStream.Output();
    LazySimpleSerDe.serialize(serializeStream, o1, oi1, serdeParams.getSeparators(), 0,
            serdeParams.getNullSequence(), serdeParams.isEscaped(), serdeParams.getEscapeChar(),
            serdeParams.getNeedsEscape());
    Text t = new Text(serializeStream.toByteArray());
    return serde.deserialize(t);
}

From source file:co.cask.cdap.hive.objectinspector.SimpleMapEqualComparerTest.java

License:Apache License

@Test
public void testIncompatibleType() throws SerDeException, IOException {
    // empty maps
    StringTextMapHolder o1 = new StringTextMapHolder();
    StructObjectInspector oi1 = (StructObjectInspector) ObjectInspectorFactory
            .getReflectionObjectInspector(StringTextMapHolder.class);

    LazySimpleSerDe serde = new LazySimpleSerDe();
    Configuration conf = new Configuration();
    Properties tbl = new Properties();
    tbl.setProperty(serdeConstants.LIST_COLUMNS, ObjectInspectorUtils.getFieldNames(oi1));
    tbl.setProperty(serdeConstants.LIST_COLUMN_TYPES, ObjectInspectorUtils.getFieldTypes(oi1));
    SerDeParameters serdeParams = LazySimpleSerDe.initSerdeParams(conf, tbl, LazySimpleSerDe.class.getName());
    serde.initialize(conf, tbl);//from w w  w  .  j  a  v a  2 s. co m
    ObjectInspector oi2 = serde.getObjectInspector();

    Object o2 = serializeAndDeserialize(o1, oi1, serde, serdeParams);

    int rc = ObjectInspectorUtils.compare(o1, oi1, o2, oi2, new SimpleMapEqualComparer());
    Assert.assertEquals(0, rc);

    // equal maps
    o1.mMap.put("42", new Text("The answer to Life, Universe And Everything"));
    o1.mMap.put("1729", new Text("A taxi cab number"));
    o2 = serializeAndDeserialize(o1, oi1, serde, serdeParams);
    rc = ObjectInspectorUtils.compare(o1, oi1, o2, oi2, new SimpleMapEqualComparer());
    Assert.assertFalse(0 == rc);
}

From source file:co.cask.cdap.hive.serde.ObjectSerializer.java

License:Apache License

public Writable serialize(Object o, ObjectInspector objectInspector) {
    //overwrite field names (as they get lost by Hive)
    StructTypeInfo structTypeInfo = (StructTypeInfo) TypeInfoUtils
            .getTypeInfoFromObjectInspector(objectInspector);
    structTypeInfo.setAllStructFieldNames(columnNames);

    List<TypeInfo> info = structTypeInfo.getAllStructFieldTypeInfos();
    List<String> names = structTypeInfo.getAllStructFieldNames();

    Map<String, Object> recordMap = new HashMap<>();
    List<Object> recordObjects = ((StructObjectInspector) objectInspector).getStructFieldsDataAsList(o);

    for (int structIndex = 0; structIndex < info.size(); structIndex++) {
        Object obj = recordObjects.get(structIndex);
        TypeInfo objType = info.get(structIndex);
        if (obj instanceof LazyNonPrimitive || obj instanceof LazyPrimitive) {
            // In case the SerDe that deserialized the object is the one of a native table
            recordMap.put(names.get(structIndex), fromLazyObject(objType, obj));
        } else if (obj instanceof Writable) {
            // Native tables sometimes introduce primitive Writable objects at this point
            recordMap.put(names.get(structIndex), fromWritable((Writable) obj));
        } else {//from   w ww. j  av  a2s . co  m
            // In case the deserializer is the DatasetSerDe
            recordMap.put(names.get(structIndex), serialize(obj, objType));
        }
    }

    // TODO Improve serialization logic - CDAP-11
    return new Text(GSON.toJson(recordMap));
}

From source file:co.cask.cdap.security.hive.HiveTokenUtils.java

License:Apache License

public static Credentials obtainToken(Credentials credentials) {
    ClassLoader hiveClassloader = ExploreUtils.getExploreClassloader();
    ClassLoader contextClassloader = Thread.currentThread().getContextClassLoader();
    Thread.currentThread().setContextClassLoader(hiveClassloader);

    try {// w ww.ja  v  a 2s.  co m
        LOG.info("Obtaining delegation token for Hive");
        Class hiveConfClass = hiveClassloader.loadClass("org.apache.hadoop.hive.conf.HiveConf");
        Object hiveConf = hiveConfClass.newInstance();

        Class hiveClass = hiveClassloader.loadClass("org.apache.hadoop.hive.ql.metadata.Hive");
        @SuppressWarnings("unchecked")
        Method hiveGet = hiveClass.getMethod("get", hiveConfClass);
        Object hiveObject = hiveGet.invoke(null, hiveConf);

        String user = UserGroupInformation.getCurrentUser().getShortUserName();
        @SuppressWarnings("unchecked")
        Method getDelegationToken = hiveClass.getMethod("getDelegationToken", String.class, String.class);
        String tokenStr = (String) getDelegationToken.invoke(hiveObject, user, user);

        Token<DelegationTokenIdentifier> delegationToken = new Token<>();
        delegationToken.decodeFromUrlString(tokenStr);
        delegationToken.setService(new Text(HiveAuthFactory.HS2_CLIENT_TOKEN));
        LOG.info("Adding delegation token {} from MetaStore for service {} for user {}", delegationToken,
                delegationToken.getService(), user);
        credentials.addToken(delegationToken.getService(), delegationToken);
        return credentials;
    } catch (Exception e) {
        LOG.error("Exception when fetching delegation token from Hive MetaStore", e);
        throw Throwables.propagate(e);
    } finally {
        Thread.currentThread().setContextClassLoader(contextClassloader);
    }
}

From source file:co.cask.cdap.security.impersonation.UGIProviderTest.java

License:Apache License

@Test
public void testRemoteUGIProvider() throws Exception {
    // Starts a mock server to handle remote UGI requests
    final NettyHttpService httpService = NettyHttpService.builder("remoteUGITest")
            .addHttpHandlers(Collections.singleton(new UGIProviderTestHandler())).build();

    httpService.startAndWait();/* w  w w  .j  a  v  a  2  s .c o m*/
    try {
        InMemoryDiscoveryService discoveryService = new InMemoryDiscoveryService();
        discoveryService
                .register(new Discoverable(Constants.Service.APP_FABRIC_HTTP, httpService.getBindAddress()));

        // Create Alice UGI
        RemoteUGIProvider ugiProvider = new RemoteUGIProvider(cConf, discoveryService, locationFactory);
        ImpersonationInfo aliceInfo = new ImpersonationInfo(getPrincipal("alice"),
                keytabFile.toURI().toString());
        UserGroupInformation aliceUGI = ugiProvider.getConfiguredUGI(aliceInfo);

        // Shouldn't be a kerberos UGI
        Assert.assertFalse(aliceUGI.hasKerberosCredentials());
        // Validate the credentials
        Token<? extends TokenIdentifier> token = aliceUGI.getCredentials().getToken(new Text("principal"));
        Assert.assertArrayEquals(aliceInfo.getPrincipal().getBytes(StandardCharsets.UTF_8),
                token.getIdentifier());
        Assert.assertArrayEquals(aliceInfo.getPrincipal().getBytes(StandardCharsets.UTF_8),
                token.getPassword());
        Assert.assertEquals(new Text("principal"), token.getKind());
        Assert.assertEquals(new Text("service"), token.getService());

        token = aliceUGI.getCredentials().getToken(new Text("keytab"));
        Assert.assertArrayEquals(aliceInfo.getKeytabURI().getBytes(StandardCharsets.UTF_8),
                token.getIdentifier());
        Assert.assertArrayEquals(aliceInfo.getKeytabURI().getBytes(StandardCharsets.UTF_8),
                token.getPassword());
        Assert.assertEquals(new Text("keytab"), token.getKind());
        Assert.assertEquals(new Text("service"), token.getService());

        // Fetch it again, it should return the same UGI due to caching
        Assert.assertSame(aliceUGI, ugiProvider.getConfiguredUGI(aliceInfo));

        // Invalid the cache and fetch it again. A different UGI should be returned
        ugiProvider.invalidCache();
        Assert.assertNotSame(aliceUGI, ugiProvider.getConfiguredUGI(aliceInfo));

    } finally {
        httpService.stopAndWait();
    }
}

From source file:co.cask.hydrator.plugin.batchSource.KafkaKey.java

License:Apache License

public long getMessageSize() {
    Text key = new Text("message.size");
    if (this.partitionMap.containsKey(key)) {
        return ((LongWritable) this.partitionMap.get(key)).get();
    } else {//  ww w.  jav  a2  s  .  co  m
        return 1024; //default estimated size
    }
}

From source file:co.cask.hydrator.plugin.batchSource.KafkaKey.java

License:Apache License

public void setMessageSize(long messageSize) {
    Text key = new Text("message.size");
    put(key, new LongWritable(messageSize));
}

From source file:co.cask.hydrator.plugin.HDFSSink.java

License:Apache License

@Override
public void transform(StructuredRecord input, Emitter<KeyValue<Text, NullWritable>> emitter) throws Exception {
    List<String> dataArray = new ArrayList<>();
    for (Schema.Field field : input.getSchema().getFields()) {
        dataArray.add(input.get(field.getName()).toString());
    }//w  w w.  ja va 2  s. co m
    emitter.emit(new KeyValue<>(new Text(Joiner.on(",").join(dataArray)), NullWritable.get()));
}

From source file:co.cask.tephra.persist.HDFSTransactionLogReaderSupplier.java

License:Apache License

public HDFSTransactionLogReaderSupplier(SequenceFile.Reader reader) {
    this.reader = reader;
    Text versionInfo = reader.getMetadata().get(new Text(TxConstants.TransactionLog.VERSION_KEY));
    this.version = versionInfo == null ? 1 : Byte.parseByte(versionInfo.toString());
}