List of usage examples for org.apache.hadoop.io Text equals
@Override public boolean equals(Object o)
o is a Text with the same contents. From source file:org.apache.rya.indexing.pcj.storage.accumulo.PcjTables.java
License:Apache License
/** * Fetch the {@link PCJMetadata} from an Accumulo table. * <p>// ww w . j a va 2 s .co m * This method assumes the PCJ table has already been created. * * @param accumuloConn - A connection to the Accumulo that hosts the PCJ table. (not null) * @param pcjTableName - The name of the table that will be search. (not null) * @return The PCJ Metadata that has been stolred in the in the PCJ Table. * @throws PCJStorageException The PCJ Table does not exist. */ public PcjMetadata getPcjMetadata(final Connector accumuloConn, final String pcjTableName) throws PCJStorageException { checkNotNull(accumuloConn); checkNotNull(pcjTableName); Scanner scanner = null; try { // Create an Accumulo scanner that iterates through the metadata entries. scanner = accumuloConn.createScanner(pcjTableName, new Authorizations()); final Iterator<Entry<Key, Value>> entries = scanner.iterator(); // No metadata has been stored in the table yet. if (!entries.hasNext()) { throw new PCJStorageException( "Could not find any PCJ metadata in the table named: " + pcjTableName); } // Fetch the metadata from the entries. Assuming they all have the same cardinality and sparql query. String sparql = null; Long cardinality = null; final Set<VariableOrder> varOrders = new HashSet<>(); while (entries.hasNext()) { final Entry<Key, Value> entry = entries.next(); final Text columnQualifier = entry.getKey().getColumnQualifier(); final byte[] value = entry.getValue().get(); if (columnQualifier.equals(PCJ_METADATA_SPARQL_QUERY)) { sparql = stringLexicoder.decode(value); } else if (columnQualifier.equals(PCJ_METADATA_CARDINALITY)) { cardinality = longLexicoder.decode(value); } else if (columnQualifier.equals(PCJ_METADATA_VARIABLE_ORDERS)) { for (final String varOrderStr : listLexicoder.decode(value)) { varOrders.add(new VariableOrder(varOrderStr)); } } } return new PcjMetadata(sparql, cardinality, varOrders); } catch (final TableNotFoundException e) { throw new PCJStorageException("Could not add results to a PCJ because the PCJ table does not exist.", e); } finally { if (scanner != null) { scanner.close(); } } }
From source file:org.apache.tez.engine.common.security.JobTokenSelector.java
License:Apache License
@SuppressWarnings("unchecked") @Override// w ww .j a va 2 s . co m public Token<JobTokenIdentifier> selectToken(Text service, Collection<Token<? extends TokenIdentifier>> tokens) { if (service == null) { return null; } for (Token<? extends TokenIdentifier> token : tokens) { if (JobTokenIdentifier.KIND_NAME.equals(token.getKind()) && service.equals(token.getService())) { return (Token<JobTokenIdentifier>) token; } } return null; }
From source file:org.culturegraph.mf.cluster.job.merge.ExplodeMapper.java
License:Apache License
@Override public void map(final Text tag, final TextArrayWritable members, final Context context) throws IOException, InterruptedException { if (tag.equals(Union.SEPARATED)) { context.getCounter(Union.UNION_FIND, Union.SEPARATED_CLASSES).increment(1); return;/*from ww w .java 2s . co m*/ } context.getCounter(Union.UNION_FIND, Union.OPEN_CLASSES).increment(1); memberSet.clear(); members.copyTo(memberSet); explode(memberSet, context); }
From source file:org.culturegraph.mf.cluster.job.merge.ResultMapper.java
License:Apache License
@Override public void map(final Text tag, final TextArrayWritable members, final Context context) throws IOException, InterruptedException { if (tag.equals(Union.OPEN)) { return;/*from ww w. ja va 2s .co m*/ } memberSet.clear(); members.copyTo(memberSet); final Text representative = memberSet.pollFirst(); for (Text member : memberSet) { final Put put = new Put(member.getBytes()); put.add(Column.Family.PROPERTY, REDIRECT, representative.getBytes()); htable.put(put); } context.getCounter(Union.UNION_FIND, "redirects written").increment(memberSet.size()); }
From source file:org.mgrover.hive.translate.GenericUDFTranslate.java
License:Apache License
/** * Pre-processes the from and to strings by calling {@link #populateMappings(Text, Text)} if * necessary.// w w w . j a va2 s . co m * * @param from * from string to be used for translation * @param to * to string to be used for translation */ private void populateMappingsIfNecessary(Text from, Text to) { // If the from and to strings haven't changed, we don't need to preprocess again to regenerate // the mappings of code points that need to replaced or deleted if ((lastFrom == null) || (lastTo == null) || !from.equals(lastFrom) || !to.equals(lastTo)) { populateMappings(from, to); // These are null when evaluate() is called for the first time if (lastFrom == null) { lastFrom = new Text(); } if (lastTo == null) { lastTo = new Text(); } // Need to deep copy here since doing something like lastFrom = from instead, will make // lastFrom point to the same Text object which would make from.equals(lastFrom) always true lastFrom.set(from); lastTo.set(to); } }
From source file:org.smartfrog.services.hadoop.benchmark.citerank.CheckingDataReducer.java
License:Open Source License
@Override public void reduce(Text key, Iterator<Text> values, OutputCollector<Text, Text> output, Reporter report) throws IOException { Set<String> links = new TreeSet<String>(); while (values.hasNext()) { Text value = values.next(); if (!value.equals(CheckingDataMapper.NONE)) { links.add(value.toString()); }//from w w w .j a v a 2s . c o m } StringBuffer sb = new StringBuffer(); for (String link : links) { sb.append(link).append("\t"); } output.collect(key, new Text(sb.toString())); }
From source file:org.teiid.translator.accumulo.AccumuloMetadataProcessor.java
License:Open Source License
public void process(MetadataFactory mf, AccumuloConnection conn) { Connector connector = conn.getInstance(); Set<String> tableNames = connector.tableOperations().list(); for (String tableName : tableNames) { try {// w w w . ja v a 2s.c om if (tableName.equals("!METADATA") || tableName.equals("trace")) { //$NON-NLS-1$ //$NON-NLS-2$ continue; } Text previousRow = null; Table table = null; Scanner scanner = connector.createScanner(tableName, conn.getAuthorizations()); for (Entry<Key, Value> entry : scanner) { Key key = entry.getKey(); Text cf = key.getColumnFamily(); Text cq = key.getColumnQualifier(); Text row = key.getRow(); if (previousRow == null || previousRow.equals(row)) { previousRow = row; if (mf.getSchema().getTable(tableName) == null) { table = mf.addTable(tableName); Column column = mf.addColumn(AccumuloMetadataProcessor.ROWID, TypeFacility.RUNTIME_NAMES.STRING, table); column.setSearchType(SearchType.All_Except_Like); mf.addPrimaryKey("PK0", Arrays.asList(AccumuloMetadataProcessor.ROWID), table); //$NON-NLS-1$ column.setUpdatable(false); } else { table = mf.getSchema().getTable(tableName); } Column column = mf.addColumn(buildColumnName(cf, cq, row), TypeFacility.RUNTIME_NAMES.STRING, table); column.setSearchType(SearchType.All_Except_Like); column.setProperty(CF, cf.toString()); column.setProperty(CQ, cq.toString()); column.setProperty(VALUE_IN, getValueIn()); column.setUpdatable(true); } else { break; } } scanner.close(); if (table != null) { table.setSupportsUpdate(true); } } catch (TableNotFoundException e) { continue; } } }
From source file:org.teiid.translator.accumulo.AccumuloUpdateExecution.java
License:Open Source License
private void performUpdate(Update update) throws TranslatorException, TableNotFoundException, MutationsRejectedException { Table table = update.getTable().getMetadataObject(); if (update.getParameterValues() != null) { throw new TranslatorException(AccumuloPlugin.Event.TEIID19005, AccumuloPlugin.Util.gs(AccumuloPlugin.Event.TEIID19005)); }//from w w w . j a v a2s . c om AccumuloQueryVisitor visitor = new AccumuloQueryVisitor(this.aef); visitor.visitNode(update.getWhere()); if (!visitor.exceptions.isEmpty()) { throw visitor.exceptions.get(0); } Connector connector = this.connection.getInstance(); BatchWriter writer = createBatchWriter(table, connector); Text prevRow = null; Iterator<Entry<Key, Value>> results = AccumuloQueryExecution.runQuery(this.aef, this.connection.getInstance(), this.connection.getAuthorizations(), visitor.getRanges(), table, visitor.scanIterators()); while (results.hasNext()) { Key key = results.next().getKey(); Text rowId = key.getRow(); if (prevRow == null || !prevRow.equals(rowId)) { prevRow = rowId; List<SetClause> changes = update.getChanges(); for (SetClause clause : changes) { Column column = clause.getSymbol().getMetadataObject(); if (SQLStringVisitor.getRecordName(column).equalsIgnoreCase(AccumuloMetadataProcessor.ROWID)) { throw new TranslatorException(AccumuloPlugin.Event.TEIID19002, AccumuloPlugin.Util.gs(AccumuloPlugin.Event.TEIID19002, table.getName())); } Expression value = clause.getValue(); if (value instanceof Literal) { Mutation mutation = buildMutation(rowId.getBytes(), column, ((Literal) value).getValue()); writer.addMutation(mutation); } else { throw new TranslatorException(AccumuloPlugin.Event.TEIID19001, AccumuloPlugin.Util.gs(AccumuloPlugin.Event.TEIID19001)); } } this.updateCount++; } } writer.close(); }
From source file:org.teiid.translator.accumulo.AccumuloUpdateExecution.java
License:Open Source License
private void performDelete(Delete delete) throws TableNotFoundException, MutationsRejectedException, TranslatorException { if (delete.getParameterValues() != null) { throw new TranslatorException(AccumuloPlugin.Event.TEIID19005, AccumuloPlugin.Util.gs(AccumuloPlugin.Event.TEIID19005)); }/* w w w . j ava2 s . c o m*/ Table table = delete.getTable().getMetadataObject(); AccumuloQueryVisitor visitor = new AccumuloQueryVisitor(this.aef); visitor.visitNode(delete.getWhere()); if (!visitor.exceptions.isEmpty()) { throw visitor.exceptions.get(0); } /* // To get the update count I am taking longer route.. Connector connector = this.connection.getInstance(); BatchDeleter deleter = connector.createBatchDeleter(SQLStringVisitor.getRecordName(table), auths, this.aef.getQueryThreadsCount(), new BatchWriterConfig()); deleter.setRanges(visitor.getRanges()); deleter.delete(); deleter.close(); */ Text prevRow = null; Connector connector = this.connection.getInstance(); BatchWriter writer = createBatchWriter(table, connector); Iterator<Entry<Key, Value>> results = AccumuloQueryExecution.runQuery(this.aef, this.connection.getInstance(), this.connection.getAuthorizations(), visitor.getRanges(), table, null); while (results.hasNext()) { Key key = results.next().getKey(); Text rowId = key.getRow(); if (prevRow == null || !prevRow.equals(rowId)) { this.updateCount++; } prevRow = rowId; Mutation mutation = new Mutation(rowId); mutation.putDelete(key.getColumnFamily(), key.getColumnQualifier()); writer.addMutation(mutation); } writer.close(); }
From source file:org.vsw.knowledge.ie.candidate.CandidateExtractor.java
License:Open Source License
private void mergeCandidate(String infile, String outfile) throws Exception { Configuration conf = HDFSUtil.getDaultConfiguration(); SequenceFileUtil<Text, Candidate> sqfile = new SequenceFileUtil<Text, Candidate>(conf, infile, Text.class, Candidate.class); SequenceFile.Reader reader = sqfile.getReader(); CandidatesWriter writer = new CandidatesWriter(outfile); Candidates current = null;/*from ww w . j av a 2 s .c om*/ Text pkey = new Text(); int entityCount = 0; while (true) { Text key = new Text(); Candidate candidate = new Candidate(); if (reader.next(key, candidate)) { if (pkey == null || !pkey.equals(key)) { if (current != null) writer.write(current); entityCount++; current = new Candidates(); } current.add(candidate); pkey = key; } else { break; } if (entityCount % 100 == 0) { System.out.println("Merge " + entityCount + " entities"); } } if (current != null) writer.write(current); writer.close(); reader.close(); }