List of usage examples for org.apache.hadoop.mapreduce JobContext getConfiguration
public Configuration getConfiguration();
From source file:org.apache.accumulo.core.client.mapreduce.AbstractInputFormat.java
License:Apache License
/** * Gets the authenticated token from either the specified token file or directly from the configuration, whichever was used when the job was configured. * * @param context//from ww w . j a v a 2 s . c o m * the Hadoop context for the configured job * @return the principal's authentication token * @since 1.6.0 * @see #setConnectorInfo(Job, String, AuthenticationToken) * @see #setConnectorInfo(Job, String, String) */ protected static AuthenticationToken getAuthenticationToken(JobContext context) { AuthenticationToken token = InputConfigurator.getAuthenticationToken(CLASS, context.getConfiguration()); return ConfiguratorBase.unwrapAuthenticationToken(context, token); }
From source file:org.apache.accumulo.core.client.mapreduce.AbstractInputFormat.java
License:Apache License
/** * Initializes an Accumulo {@link org.apache.accumulo.core.client.Instance} based on the configuration. * * @param context/* ww w. j a va 2 s . co m*/ * the Hadoop context for the configured job * @return an Accumulo instance * @since 1.5.0 * @see #setZooKeeperInstance(Job, ClientConfiguration) */ protected static Instance getInstance(JobContext context) { return InputConfigurator.getInstance(CLASS, context.getConfiguration()); }
From source file:org.apache.accumulo.core.client.mapreduce.AbstractInputFormat.java
License:Apache License
/** * Gets the log level from this configuration. * * @param context// ww w . j a va 2 s. c om * the Hadoop context for the configured job * @return the log level * @since 1.5.0 * @see #setLogLevel(Job, Level) */ protected static Level getLogLevel(JobContext context) { return InputConfigurator.getLogLevel(CLASS, context.getConfiguration()); }
From source file:org.apache.accumulo.core.client.mapreduce.AbstractInputFormat.java
License:Apache License
/** * Gets the authorizations to set for the scans from the configuration. * * @param context//w w w .ja v a 2s .co m * the Hadoop context for the configured job * @return the Accumulo scan authorizations * @since 1.5.0 * @see #setScanAuthorizations(Job, Authorizations) */ protected static Authorizations getScanAuthorizations(JobContext context) { return InputConfigurator.getScanAuthorizations(CLASS, context.getConfiguration()); }
From source file:org.apache.accumulo.core.client.mapreduce.AbstractInputFormat.java
License:Apache License
/** * Fetches all {@link InputTableConfig}s that have been set on the given job. * * @param context/*from w ww . j av a 2s.c om*/ * the Hadoop job instance to be configured * @return the {@link InputTableConfig} objects for the job * @since 1.6.0 */ protected static Map<String, InputTableConfig> getInputTableConfigs(JobContext context) { return InputConfigurator.getInputTableConfigs(CLASS, context.getConfiguration()); }
From source file:org.apache.accumulo.core.client.mapreduce.AbstractInputFormat.java
License:Apache License
/** * Fetches a {@link InputTableConfig} that has been set on the configuration for a specific table. * * <p>//from ww w . ja v a2s. co m * null is returned in the event that the table doesn't exist. * * @param context * the Hadoop job instance to be configured * @param tableName * the table name for which to grab the config object * @return the {@link InputTableConfig} for the given table * @since 1.6.0 */ protected static InputTableConfig getInputTableConfig(JobContext context, String tableName) { return InputConfigurator.getInputTableConfig(CLASS, context.getConfiguration(), tableName); }
From source file:org.apache.accumulo.core.client.mapreduce.AbstractInputFormat.java
License:Apache License
/** * Initializes an Accumulo {@link org.apache.accumulo.core.client.impl.TabletLocator} based on the configuration. * * @param context//from w w w.j a v a 2s . co m * the Hadoop context for the configured job * @param table * the table for which to initialize the locator * @return an Accumulo tablet locator * @throws org.apache.accumulo.core.client.TableNotFoundException * if the table name set on the configuration doesn't exist * @since 1.6.0 * @deprecated since 1.7.0 This method returns a type that is not part of the public API and is not guaranteed to be stable. The method was deprecated to * discourage its use. */ @Deprecated protected static TabletLocator getTabletLocator(JobContext context, String table) throws TableNotFoundException { return InputConfigurator.getTabletLocator(CLASS, context.getConfiguration(), table); }
From source file:org.apache.accumulo.core.client.mapreduce.AbstractInputFormat.java
License:Apache License
/** * Check whether a configuration is fully configured to be used with an Accumulo {@link org.apache.hadoop.mapreduce.InputFormat}. * * @param context/*from w w w. jav a2s. c o m*/ * the Hadoop context for the configured job * @throws java.io.IOException * if the context is improperly configured * @since 1.5.0 */ protected static void validateOptions(JobContext context) throws IOException { final Configuration conf = context.getConfiguration(); final Instance inst = InputConfigurator.validateInstance(CLASS, conf); String principal = InputConfigurator.getPrincipal(CLASS, conf); AuthenticationToken token = InputConfigurator.getAuthenticationToken(CLASS, conf); // In secure mode, we need to convert the DelegationTokenStub into a real DelegationToken token = ConfiguratorBase.unwrapAuthenticationToken(context, token); Connector conn; try { conn = inst.getConnector(principal, token); } catch (Exception e) { throw new IOException(e); } InputConfigurator.validatePermissions(CLASS, conf, conn); }
From source file:org.apache.accumulo.core.client.mapreduce.AbstractInputFormat.java
License:Apache License
/** * Construct the {@link ClientConfiguration} given the provided context. * * @param context/*from ww w. j av a 2 s. c o m*/ * The Job * @return The ClientConfiguration * @since 1.7.0 */ protected static ClientConfiguration getClientConfiguration(JobContext context) { return InputConfigurator.getClientConfiguration(CLASS, context.getConfiguration()); }
From source file:org.apache.accumulo.core.client.mapreduce.AbstractInputFormat.java
License:Apache License
/** * Gets the splits of the tables that have been set on the job by reading the metadata table for the specified ranges. * * @return the splits from the tables based on the ranges. * @throws java.io.IOException/*w w w. jav a 2 s . co m*/ * if a table set on the job doesn't exist or an error occurs initializing the tablet locator */ @Override public List<InputSplit> getSplits(JobContext context) throws IOException { Level logLevel = getLogLevel(context); log.setLevel(logLevel); validateOptions(context); Random random = new Random(); LinkedList<InputSplit> splits = new LinkedList<>(); Map<String, InputTableConfig> tableConfigs = getInputTableConfigs(context); for (Map.Entry<String, InputTableConfig> tableConfigEntry : tableConfigs.entrySet()) { String tableName = tableConfigEntry.getKey(); InputTableConfig tableConfig = tableConfigEntry.getValue(); Instance instance = getInstance(context); String tableId; // resolve table name to id once, and use id from this point forward if (DeprecationUtil.isMockInstance(instance)) { tableId = ""; } else { try { tableId = Tables.getTableId(instance, tableName); } catch (TableNotFoundException e) { throw new IOException(e); } } Authorizations auths = getScanAuthorizations(context); String principal = getPrincipal(context); AuthenticationToken token = getAuthenticationToken(context); boolean batchScan = InputConfigurator.isBatchScan(CLASS, context.getConfiguration()); boolean supportBatchScan = !(tableConfig.isOfflineScan() || tableConfig.shouldUseIsolatedScanners() || tableConfig.shouldUseLocalIterators()); if (batchScan && !supportBatchScan) throw new IllegalArgumentException( "BatchScanner optimization not available for offline scan, isolated, or local iterators"); boolean autoAdjust = tableConfig.shouldAutoAdjustRanges(); if (batchScan && !autoAdjust) throw new IllegalArgumentException( "AutoAdjustRanges must be enabled when using BatchScanner optimization"); List<Range> ranges = autoAdjust ? Range.mergeOverlapping(tableConfig.getRanges()) : tableConfig.getRanges(); if (ranges.isEmpty()) { ranges = new ArrayList<>(1); ranges.add(new Range()); } // get the metadata information for these ranges Map<String, Map<KeyExtent, List<Range>>> binnedRanges = new HashMap<>(); TabletLocator tl; try { if (tableConfig.isOfflineScan()) { binnedRanges = binOfflineTable(context, tableId, ranges); while (binnedRanges == null) { // Some tablets were still online, try again // sleep randomly between 100 and 200 ms sleepUninterruptibly(100 + random.nextInt(100), TimeUnit.MILLISECONDS); binnedRanges = binOfflineTable(context, tableId, ranges); } } else { tl = InputConfigurator.getTabletLocator(CLASS, context.getConfiguration(), tableId); // its possible that the cache could contain complete, but old information about a tables tablets... so clear it tl.invalidateCache(); ClientContext clientContext = new ClientContext(getInstance(context), new Credentials(getPrincipal(context), getAuthenticationToken(context)), getClientConfiguration(context)); while (!tl.binRanges(clientContext, ranges, binnedRanges).isEmpty()) { if (!DeprecationUtil.isMockInstance(instance)) { if (!Tables.exists(instance, tableId)) throw new TableDeletedException(tableId); if (Tables.getTableState(instance, tableId) == TableState.OFFLINE) throw new TableOfflineException(instance, tableId); } binnedRanges.clear(); log.warn("Unable to locate bins for specified ranges. Retrying."); // sleep randomly between 100 and 200 ms sleepUninterruptibly(100 + random.nextInt(100), TimeUnit.MILLISECONDS); tl.invalidateCache(); } } } catch (Exception e) { throw new IOException(e); } // all of this code will add either range per each locations or split ranges and add range-location split // Map from Range to Array of Locations, we only use this if we're don't split HashMap<Range, ArrayList<String>> splitsToAdd = null; if (!autoAdjust) splitsToAdd = new HashMap<>(); HashMap<String, String> hostNameCache = new HashMap<>(); for (Map.Entry<String, Map<KeyExtent, List<Range>>> tserverBin : binnedRanges.entrySet()) { String ip = tserverBin.getKey().split(":", 2)[0]; String location = hostNameCache.get(ip); if (location == null) { InetAddress inetAddress = InetAddress.getByName(ip); location = inetAddress.getCanonicalHostName(); hostNameCache.put(ip, location); } for (Map.Entry<KeyExtent, List<Range>> extentRanges : tserverBin.getValue().entrySet()) { Range ke = extentRanges.getKey().toDataRange(); if (batchScan) { // group ranges by tablet to be read by a BatchScanner ArrayList<Range> clippedRanges = new ArrayList<>(); for (Range r : extentRanges.getValue()) clippedRanges.add(ke.clip(r)); BatchInputSplit split = new BatchInputSplit(tableName, tableId, clippedRanges, new String[] { location }); SplitUtils.updateSplit(split, instance, tableConfig, principal, token, auths, logLevel); splits.add(split); } else { // not grouping by tablet for (Range r : extentRanges.getValue()) { if (autoAdjust) { // divide ranges into smaller ranges, based on the tablets RangeInputSplit split = new RangeInputSplit(tableName, tableId, ke.clip(r), new String[] { location }); SplitUtils.updateSplit(split, instance, tableConfig, principal, token, auths, logLevel); split.setOffline(tableConfig.isOfflineScan()); split.setIsolatedScan(tableConfig.shouldUseIsolatedScanners()); split.setUsesLocalIterators(tableConfig.shouldUseLocalIterators()); splits.add(split); } else { // don't divide ranges ArrayList<String> locations = splitsToAdd.get(r); if (locations == null) locations = new ArrayList<>(1); locations.add(location); splitsToAdd.put(r, locations); } } } } } if (!autoAdjust) for (Map.Entry<Range, ArrayList<String>> entry : splitsToAdd.entrySet()) { RangeInputSplit split = new RangeInputSplit(tableName, tableId, entry.getKey(), entry.getValue().toArray(new String[0])); SplitUtils.updateSplit(split, instance, tableConfig, principal, token, auths, logLevel); split.setOffline(tableConfig.isOfflineScan()); split.setIsolatedScan(tableConfig.shouldUseIsolatedScanners()); split.setUsesLocalIterators(tableConfig.shouldUseLocalIterators()); splits.add(split); } } return splits; }