List of usage examples for org.w3c.dom Element getTagName
public String getTagName();
From source file:org.apache.hadoop.mapred.PoolManager.java
/** * Updates the allocation list from the allocation config file. This file is * expected to be in the following whitespace-separated format: * /*from w w w. j a v a2 s .c o m*/ * <code> * poolName1 mapAlloc reduceAlloc * poolName2 mapAlloc reduceAlloc * ... * </code> * * Blank lines and lines starting with # are ignored. * * @throws IOException if the config file cannot be read. * @throws AllocationConfigurationException if allocations are invalid. * @throws ParserConfigurationException if XML parser is misconfigured. * @throws SAXException if config file is malformed. */ public void reloadAllocs() throws IOException, ParserConfigurationException, SAXException, AllocationConfigurationException { if (allocFile == null) return; // Create some temporary hashmaps to hold the new allocs, and we only save // them in our fields if we have parsed the entire allocs file successfully. Map<String, Integer> mapAllocs = new HashMap<String, Integer>(); Map<String, Integer> reduceAllocs = new HashMap<String, Integer>(); Map<String, Integer> poolMaxJobs = new HashMap<String, Integer>(); Map<String, Integer> userMaxJobs = new HashMap<String, Integer>(); Map<String, Integer> poolMaxMaps = new HashMap<String, Integer>(); Map<String, Integer> poolMaxReduces = new HashMap<String, Integer>(); Map<String, Double> poolWeights = new HashMap<String, Double>(); Map<String, SchedulingMode> poolModes = new HashMap<String, SchedulingMode>(); Map<String, Long> minSharePreemptionTimeouts = new HashMap<String, Long>(); int userMaxJobsDefault = Integer.MAX_VALUE; int poolMaxJobsDefault = Integer.MAX_VALUE; long fairSharePreemptionTimeout = Long.MAX_VALUE; long defaultMinSharePreemptionTimeout = Long.MAX_VALUE; SchedulingMode defaultSchedulingMode = SchedulingMode.FAIR; // Remember all pool names so we can display them on web UI, etc. List<String> poolNamesInAllocFile = new ArrayList<String>(); // Read and parse the allocations file. DocumentBuilderFactory docBuilderFactory = DocumentBuilderFactory.newInstance(); docBuilderFactory.setIgnoringComments(true); DocumentBuilder builder = docBuilderFactory.newDocumentBuilder(); Document doc; if (allocFile instanceof String) { doc = builder.parse(new File((String) allocFile)); } else { doc = builder.parse(allocFile.toString()); } Element root = doc.getDocumentElement(); if (!"allocations".equals(root.getTagName())) throw new AllocationConfigurationException( "Bad fair scheduler config " + "file: top-level element not <allocations>"); NodeList elements = root.getChildNodes(); for (int i = 0; i < elements.getLength(); i++) { Node node = elements.item(i); if (!(node instanceof Element)) continue; Element element = (Element) node; if ("pool".equals(element.getTagName())) { String poolName = element.getAttribute("name"); poolNamesInAllocFile.add(poolName); NodeList fields = element.getChildNodes(); for (int j = 0; j < fields.getLength(); j++) { Node fieldNode = fields.item(j); if (!(fieldNode instanceof Element)) continue; Element field = (Element) fieldNode; if ("minMaps".equals(field.getTagName())) { String text = ((Text) field.getFirstChild()).getData().trim(); int val = Integer.parseInt(text); mapAllocs.put(poolName, val); } else if ("minReduces".equals(field.getTagName())) { String text = ((Text) field.getFirstChild()).getData().trim(); int val = Integer.parseInt(text); reduceAllocs.put(poolName, val); } else if ("maxMaps".equals(field.getTagName())) { String text = ((Text) field.getFirstChild()).getData().trim(); int val = Integer.parseInt(text); poolMaxMaps.put(poolName, val); } else if ("maxReduces".equals(field.getTagName())) { String text = ((Text) field.getFirstChild()).getData().trim(); int val = Integer.parseInt(text); poolMaxReduces.put(poolName, val); } else if ("maxRunningJobs".equals(field.getTagName())) { String text = ((Text) field.getFirstChild()).getData().trim(); int val = Integer.parseInt(text); poolMaxJobs.put(poolName, val); } else if ("weight".equals(field.getTagName())) { String text = ((Text) field.getFirstChild()).getData().trim(); double val = Double.parseDouble(text); poolWeights.put(poolName, val); } else if ("minSharePreemptionTimeout".equals(field.getTagName())) { String text = ((Text) field.getFirstChild()).getData().trim(); long val = Long.parseLong(text) * 1000L; minSharePreemptionTimeouts.put(poolName, val); } else if ("schedulingMode".equals(field.getTagName())) { String text = ((Text) field.getFirstChild()).getData().trim(); poolModes.put(poolName, parseSchedulingMode(text)); } } if (poolMaxMaps.containsKey(poolName) && mapAllocs.containsKey(poolName) && poolMaxMaps.get(poolName) < mapAllocs.get(poolName)) { LOG.warn(String.format("Pool %s has max maps %d less than min maps %d", poolName, poolMaxMaps.get(poolName), mapAllocs.get(poolName))); } if (poolMaxReduces.containsKey(poolName) && reduceAllocs.containsKey(poolName) && poolMaxReduces.get(poolName) < reduceAllocs.get(poolName)) { LOG.warn(String.format("Pool %s has max reduces %d less than min reduces %d", poolName, poolMaxReduces.get(poolName), reduceAllocs.get(poolName))); } } else if ("user".equals(element.getTagName())) { String userName = element.getAttribute("name"); NodeList fields = element.getChildNodes(); for (int j = 0; j < fields.getLength(); j++) { Node fieldNode = fields.item(j); if (!(fieldNode instanceof Element)) continue; Element field = (Element) fieldNode; if ("maxRunningJobs".equals(field.getTagName())) { String text = ((Text) field.getFirstChild()).getData().trim(); int val = Integer.parseInt(text); userMaxJobs.put(userName, val); } } } else if ("userMaxJobsDefault".equals(element.getTagName())) { String text = ((Text) element.getFirstChild()).getData().trim(); int val = Integer.parseInt(text); userMaxJobsDefault = val; } else if ("poolMaxJobsDefault".equals(element.getTagName())) { String text = ((Text) element.getFirstChild()).getData().trim(); int val = Integer.parseInt(text); poolMaxJobsDefault = val; } else if ("fairSharePreemptionTimeout".equals(element.getTagName())) { String text = ((Text) element.getFirstChild()).getData().trim(); long val = Long.parseLong(text) * 1000L; fairSharePreemptionTimeout = val; } else if ("defaultMinSharePreemptionTimeout".equals(element.getTagName())) { String text = ((Text) element.getFirstChild()).getData().trim(); long val = Long.parseLong(text) * 1000L; defaultMinSharePreemptionTimeout = val; } else if ("defaultPoolSchedulingMode".equals(element.getTagName())) { String text = ((Text) element.getFirstChild()).getData().trim(); defaultSchedulingMode = parseSchedulingMode(text); } else { LOG.warn("Bad element in allocations file: " + element.getTagName()); } } // Commit the reload; also create any pool defined in the alloc file // if it does not already exist, so it can be displayed on the web UI. synchronized (this) { this.mapAllocs = mapAllocs; this.reduceAllocs = reduceAllocs; this.poolMaxMaps = poolMaxMaps; this.poolMaxReduces = poolMaxReduces; this.poolMaxJobs = poolMaxJobs; this.userMaxJobs = userMaxJobs; this.poolWeights = poolWeights; this.minSharePreemptionTimeouts = minSharePreemptionTimeouts; this.userMaxJobsDefault = userMaxJobsDefault; this.poolMaxJobsDefault = poolMaxJobsDefault; this.fairSharePreemptionTimeout = fairSharePreemptionTimeout; this.defaultMinSharePreemptionTimeout = defaultMinSharePreemptionTimeout; this.defaultSchedulingMode = defaultSchedulingMode; for (String name : poolNamesInAllocFile) { Pool pool = getPool(name); if (poolModes.containsKey(name)) { pool.setSchedulingMode(poolModes.get(name)); } else { pool.setSchedulingMode(defaultSchedulingMode); } } } }
From source file:org.apache.hadoop.mapred.QueueConfigurationParser.java
private Queue parseResource(Element queuesNode) { Queue rootNode = null;/*from ww w. j a v a 2 s . c o m*/ try { if (!QUEUES_TAG.equals(queuesNode.getTagName())) { LOG.info("Bad conf file: top-level element not <queues>"); throw new RuntimeException("No queues defined "); } NamedNodeMap nmp = queuesNode.getAttributes(); Node acls = nmp.getNamedItem(ACLS_ENABLED_TAG); if (acls != null) { LOG.warn("Configuring " + ACLS_ENABLED_TAG + " flag in " + QueueManager.QUEUE_CONF_FILE_NAME + " is not valid. " + "This tag is ignored. Configure " + MRConfig.MR_ACLS_ENABLED + " in mapred-site.xml. See the " + " documentation of " + MRConfig.MR_ACLS_ENABLED + ", which is used for enabling job level authorization and " + " queue level authorization."); } NodeList props = queuesNode.getChildNodes(); if (props == null || props.getLength() <= 0) { LOG.info(" Bad configuration no queues defined "); throw new RuntimeException(" No queues defined "); } //We have root level nodes. for (int i = 0; i < props.getLength(); i++) { Node propNode = props.item(i); if (!(propNode instanceof Element)) { continue; } if (!propNode.getNodeName().equals(QUEUE_TAG)) { LOG.info("At root level only \" queue \" tags are allowed "); throw new RuntimeException("Malformed xml document no queue defined "); } Element prop = (Element) propNode; //Add children to root. Queue q = createHierarchy("", prop); if (rootNode == null) { rootNode = new Queue(); rootNode.setName(""); } rootNode.addChild(q); } return rootNode; } catch (DOMException e) { LOG.info("Error parsing conf file: " + e); throw new RuntimeException(e); } }
From source file:org.apache.hadoop.mapred.QueueConfigurationParser.java
/** * @param parent Name of the parent queue * @param queueNode//from www . jav a2 s .c om * @return */ private Queue createHierarchy(String parent, Element queueNode) { if (queueNode == null) { return null; } //Name of the current queue. //Complete qualified queue name. String name = ""; Queue newQueue = new Queue(); Map<String, AccessControlList> acls = new HashMap<String, AccessControlList>(); NodeList fields = queueNode.getChildNodes(); validate(queueNode); List<Element> subQueues = new ArrayList<Element>(); String submitKey = ""; String adminKey = ""; for (int j = 0; j < fields.getLength(); j++) { Node fieldNode = fields.item(j); if (!(fieldNode instanceof Element)) { continue; } Element field = (Element) fieldNode; if (QUEUE_NAME_TAG.equals(field.getTagName())) { String nameValue = field.getTextContent(); if (field.getTextContent() == null || field.getTextContent().trim().equals("") || field.getTextContent().contains(NAME_SEPARATOR)) { throw new RuntimeException("Improper queue name : " + nameValue); } if (!parent.equals("")) { name += parent + NAME_SEPARATOR; } //generate the complete qualified name //parent.child name += nameValue; newQueue.setName(name); submitKey = toFullPropertyName(name, QueueACL.SUBMIT_JOB.getAclName()); adminKey = toFullPropertyName(name, QueueACL.ADMINISTER_JOBS.getAclName()); } if (QUEUE_TAG.equals(field.getTagName()) && field.hasChildNodes()) { subQueues.add(field); } if (isAclsEnabled()) { if (ACL_SUBMIT_JOB_TAG.equals(field.getTagName())) { acls.put(submitKey, new AccessControlList(field.getTextContent())); } if (ACL_ADMINISTER_JOB_TAG.equals(field.getTagName())) { acls.put(adminKey, new AccessControlList(field.getTextContent())); } } if (PROPERTIES_TAG.equals(field.getTagName())) { Properties properties = populateProperties(field); newQueue.setProperties(properties); } if (STATE_TAG.equals(field.getTagName())) { String state = field.getTextContent(); newQueue.setState(QueueState.getState(state)); } } if (!acls.containsKey(submitKey)) { acls.put(submitKey, new AccessControlList(" ")); } if (!acls.containsKey(adminKey)) { acls.put(adminKey, new AccessControlList(" ")); } //Set acls newQueue.setAcls(acls); //At this point we have the queue ready at current height level. //so we have parent name available. for (Element field : subQueues) { newQueue.addChild(createHierarchy(newQueue.getName(), field)); } return newQueue; }
From source file:org.apache.hadoop.mapred.TaskErrorCollector.java
private static boolean matched(Element element, String tagName) { return tagName.equals(element.getTagName()); }
From source file:org.apache.hadoop.mapred.YTPoolManager.java
/** * Updates the allocation list from the allocation config file. This file is * expected to be in the following whitespace-separated format: * //ww w . ja v a2 s . c om * <code> * poolName1 mapAlloc reduceAlloc * poolName2 mapAlloc reduceAlloc * ... * </code> * * Blank lines and lines starting with # are ignored. * * @throws IOException if the config file cannot be read. * @throws YTAllocationConfigurationException if allocations are invalid. * @throws ParserConfigurationException if XML parser is misconfigured. * @throws SAXException if config file is malformed. */ public void reloadAllocs() throws IOException, ParserConfigurationException, SAXException, YTAllocationConfigurationException { if (allocFile == null) return; // Create some temporary hashmaps to hold the new allocs, and we only save // them in our fields if we have parsed the entire allocs file successfully. Map<String, Integer> mapAllocs = new HashMap<String, Integer>(); Map<String, Integer> reduceAllocs = new HashMap<String, Integer>(); Map<String, Integer> poolMaxJobs = new HashMap<String, Integer>(); Map<String, Integer> userMaxJobs = new HashMap<String, Integer>(); Map<String, Integer> poolWeights = new HashMap<String, Integer>(); Map<String, Boolean> poolUseFIFO = new HashMap<String, Boolean>(); Map<String, ReduceWatcher> poolReduceWatcher = new HashMap<String, ReduceWatcher>(); int userMaxJobsDefault = POOL_MAX_JOBS_DEFAULT; // Remember all pool names so we can display them on web UI, etc. List<String> poolNamesInAllocFile = new ArrayList<String>(); // Read and parse the allocations file. DocumentBuilderFactory docBuilderFactory = DocumentBuilderFactory.newInstance(); docBuilderFactory.setIgnoringComments(true); DocumentBuilder builder = docBuilderFactory.newDocumentBuilder(); Document doc = builder.parse(new File(allocFile)); Element root = doc.getDocumentElement(); if (!"allocations".equals(root.getTagName())) throw new YTAllocationConfigurationException( "Bad allocations file: " + "top-level element not <allocations>"); NodeList elements = root.getChildNodes(); for (int i = 0; i < elements.getLength(); i++) { Node node = elements.item(i); if (!(node instanceof Element)) continue; Element element = (Element) node; if ("pool".equals(element.getTagName())) { String poolName = element.getAttribute("name"); poolNamesInAllocFile.add(poolName); NodeList fields = element.getChildNodes(); for (int j = 0; j < fields.getLength(); j++) { Node fieldNode = fields.item(j); if (!(fieldNode instanceof Element)) continue; Element field = (Element) fieldNode; if ("minMaps".equals(field.getTagName())) { String text = ((Text) field.getFirstChild()).getData().trim(); int val = Integer.parseInt(text); mapAllocs.put(poolName, val); } else if ("minReduces".equals(field.getTagName())) { String text = ((Text) field.getFirstChild()).getData().trim(); int val = Integer.parseInt(text); reduceAllocs.put(poolName, val); } else if ("maxRunningJobs".equals(field.getTagName())) { String text = ((Text) field.getFirstChild()).getData().trim(); int val = Integer.parseInt(text); poolMaxJobs.put(poolName, val); } else if ("weight".equals(field.getTagName())) { String text = ((Text) field.getFirstChild()).getData().trim(); int val = Integer.parseInt(text); poolWeights.put(poolName, val); } else if ("useFIFO".equals(field.getTagName())) { String text = ((Text) field.getFirstChild()).getData().trim(); boolean val = Boolean.parseBoolean(text); poolUseFIFO.put(poolName, val); } else if ("lazyReduce".equals(field.getTagName())) { String text = ((Text) field.getFirstChild()).getData().trim(); ReduceWatcher watcher = new ReduceWatcher(text); poolReduceWatcher.put(poolName, watcher); } } } else if ("user".equals(element.getTagName())) { String userName = element.getAttribute("name"); NodeList fields = element.getChildNodes(); for (int j = 0; j < fields.getLength(); j++) { Node fieldNode = fields.item(j); if (!(fieldNode instanceof Element)) continue; Element field = (Element) fieldNode; if ("maxRunningJobs".equals(field.getTagName())) { String text = ((Text) field.getFirstChild()).getData().trim(); int val = Integer.parseInt(text); userMaxJobs.put(userName, val); } } } else if ("userMaxJobsDefault".equals(element.getTagName())) { String text = ((Text) element.getFirstChild()).getData().trim(); int val = Integer.parseInt(text); userMaxJobsDefault = val; } else { LOG.warn("Bad element in allocations file: " + element.getTagName()); } } // Commit the reload; also create any pool defined in the alloc file // if it does not already exist, so it can be displayed on the web UI. synchronized (this) { this.mapAllocs = mapAllocs; this.reduceAllocs = reduceAllocs; this.poolMaxJobs = poolMaxJobs; this.userMaxJobs = userMaxJobs; this.userMaxJobsDefault = userMaxJobsDefault; this.poolWeights = poolWeights; this.poolUseFIFO = poolUseFIFO; this.poolReduceWatcher = poolReduceWatcher; // remove deleted queue Set<String> uncontainedPoolNameSet = new HashSet<String>(); for (String key : pools.keySet()) uncontainedPoolNameSet.add(key); for (String name : poolNamesInAllocFile.toArray(new String[0])) { if (uncontainedPoolNameSet.contains(name)) { uncontainedPoolNameSet.remove(name); } } for (String name : uncontainedPoolNameSet) { if (getPool(name).getJobs().size() == 0) pools.remove(name); } uncontainedPoolNameSet = null; for (String name : poolNamesInAllocFile) { getPool(name); } // create a list of ordered pools to be scheduled. this.orderedPools = orderPools(); this.poolIndexLastAccessed = 0; } }
From source file:org.apache.hadoop.mapreduce.v2.app.rm.HostGroupLabelsFileReader.java
private static void readXMLFileToMap(String labelsFile, Map<String, Set<String>> hostGroupLabelsMap) throws IOException { try {/*from w w w .j a v a2 s.c o m*/ URL url = Thread.currentThread().getContextClassLoader().getResource(labelsFile); if (url == null) { LOG.fatal("Host group labels file " + labelsFile + " not found"); throw new RuntimeException("Host group labels file " + labelsFile + " not found"); } DocumentBuilderFactory dbFactory = DocumentBuilderFactory.newInstance(); DocumentBuilder docBuilder = dbFactory.newDocumentBuilder(); Document doc = docBuilder.parse(url.toString()); if (doc == null) { LOG.fatal("Error parsing host group labels file " + labelsFile); throw new RuntimeException("Error parsing host group labels file " + labelsFile); } Element root = doc.getDocumentElement(); root.normalize(); if (!"CBD-NodeGroups".equalsIgnoreCase(root.getTagName())) { LOG.fatal("Bad node group labels file " + labelsFile); throw new RuntimeException("Bad node group labels file " + labelsFile); } NodeList nodeGroupLabelsList = root.getChildNodes(); for (int i = 0; i < nodeGroupLabelsList.getLength(); i++) { Node nodeGroupLabelNode = nodeGroupLabelsList.item(i); if (!(nodeGroupLabelNode instanceof Element)) continue; Element nodeGroupLabel = (Element) nodeGroupLabelNode; if (!"NodeGroupLabel".equalsIgnoreCase(nodeGroupLabel.getTagName())) { throw new RuntimeException("Bad CBD NodeGroups labels file - no NodeGroupLabel"); } String nodeGroupLabelAttr = nodeGroupLabel.getAttribute("label"); NodeList nodeList = nodeGroupLabel.getChildNodes(); Set<String> nodeSet = new HashSet<String>(); for (int j = 0; j < nodeList.getLength(); j++) { Node nodeNode = nodeList.item(j); if (!(nodeNode instanceof Element)) continue; Element nodeElem = (Element) nodeNode; if ("Node".equalsIgnoreCase(nodeElem.getTagName())) { nodeSet.add(nodeElem.getTextContent().trim()); } } hostGroupLabelsMap.put(nodeGroupLabelAttr.toLowerCase(), nodeSet); } } catch (SAXException e) { LOG.fatal("error parsing conf file: " + e); throw new RuntimeException(e); } catch (ParserConfigurationException e) { LOG.fatal("error parsing conf file: " + e); throw new RuntimeException(e); } }
From source file:org.apache.hadoop.raid.ConfigManager.java
void reloadXmlConfigs() throws IOException, ParserConfigurationException, SAXException, ClassNotFoundException, RaidConfigurationException {/*from ww w .j a v a 2 s . c om*/ if (configFileName == null) { return; } File file = new File(configFileName); if (!file.exists()) { throw new RaidConfigurationException("Configuration file " + configFileName + " does not exist."); } // Create some temporary hashmaps to hold the new allocs, and we only save // them in our fields if we have parsed the entire allocs file successfully. List<PolicyInfo> all = new ArrayList<PolicyInfo>(); long periodicityValue = periodicity; // Read and parse the configuration file. // allow include files in configuration file DocumentBuilderFactory docBuilderFactory = DocumentBuilderFactory.newInstance(); docBuilderFactory.setIgnoringComments(true); docBuilderFactory.setNamespaceAware(true); try { docBuilderFactory.setXIncludeAware(true); } catch (UnsupportedOperationException e) { LOG.error("Failed to set setXIncludeAware(true) for raid parser " + docBuilderFactory + ":" + e, e); } LOG.info("Reloading config file " + file); DocumentBuilder builder = docBuilderFactory.newDocumentBuilder(); Document doc = builder.parse(file); Element root = doc.getDocumentElement(); if (!"configuration".equalsIgnoreCase(root.getTagName())) throw new RaidConfigurationException( "Bad configuration file: " + "top-level element not <configuration>"); NodeList policies = root.getChildNodes(); Map<String, PolicyInfo> existingPolicies = new HashMap<String, PolicyInfo>(); // loop through all the configured policies. for (int i = 0; i < policies.getLength(); i++) { Node node = policies.item(i); if (!(node instanceof Element)) { continue; } Element policy = (Element) node; if ("policy".equalsIgnoreCase(policy.getTagName())) { String policyName = policy.getAttribute("name"); PolicyInfo curr = new PolicyInfo(policyName, conf); PolicyInfo parent = null; NodeList policyElements = policy.getChildNodes(); for (int j = 0; j < policyElements.getLength(); j++) { Node node1 = policyElements.item(j); if (!(node1 instanceof Element)) { continue; } Element property = (Element) node1; String propertyName = property.getTagName(); if ("srcPath".equalsIgnoreCase(propertyName)) { String srcPathPrefix = property.getAttribute("prefix"); if (srcPathPrefix != null && srcPathPrefix.length() > 0) { curr.setSrcPath(srcPathPrefix); } } else if ("fileList".equalsIgnoreCase(propertyName)) { String text = ((Text) property.getFirstChild()).getData().trim(); LOG.info(policyName + ".fileList = " + text); curr.setFileListPath(new Path(text)); } else if ("codecId".equalsIgnoreCase(propertyName)) { String text = ((Text) property.getFirstChild()).getData().trim(); LOG.info(policyName + ".codecId = " + text); curr.setCodecId(text); } else if ("shouldRaid".equalsIgnoreCase(propertyName)) { String text = ((Text) property.getFirstChild()).getData().trim(); curr.setShouldRaid(Boolean.parseBoolean(text)); } else if ("description".equalsIgnoreCase(propertyName)) { String text = ((Text) property.getFirstChild()).getData().trim(); curr.setDescription(text); } else if ("parentPolicy".equalsIgnoreCase(propertyName)) { String text = ((Text) property.getFirstChild()).getData().trim(); parent = existingPolicies.get(text); } else if ("property".equalsIgnoreCase(propertyName)) { NodeList nl = property.getChildNodes(); String pname = null, pvalue = null; for (int l = 0; l < nl.getLength(); l++) { Node node3 = nl.item(l); if (!(node3 instanceof Element)) { continue; } Element item = (Element) node3; String itemName = item.getTagName(); if ("name".equalsIgnoreCase(itemName)) { pname = ((Text) item.getFirstChild()).getData().trim(); } else if ("value".equalsIgnoreCase(itemName)) { pvalue = ((Text) item.getFirstChild()).getData().trim(); } } if (pname != null && pvalue != null) { LOG.info(policyName + "." + pname + " = " + pvalue); curr.setProperty(pname, pvalue); } } else { LOG.info("Found bad property " + propertyName + " policy name " + policyName + ". Ignoring."); } } // done with all properties of this policy PolicyInfo pinfo; if (parent != null) { pinfo = new PolicyInfo(policyName, conf); pinfo.copyFrom(parent); pinfo.copyFrom(curr); } else { pinfo = curr; } if (pinfo.getSrcPath() != null || pinfo.getFileListPath() != null) { all.add(pinfo); } existingPolicies.put(policyName, pinfo); } } setAllPolicies(all); periodicity = periodicityValue; return; }
From source file:org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.AllocationFileLoaderService.java
/** * Updates the allocation list from the allocation config file. This file is * expected to be in the XML format specified in the design doc. * * @throws IOException if the config file cannot be read. * @throws AllocationConfigurationException if allocations are invalid. * @throws ParserConfigurationException if XML parser is misconfigured. * @throws SAXException if config file is malformed. *//* w w w.j av a 2 s. co m*/ public synchronized void reloadAllocations() throws IOException, ParserConfigurationException, SAXException, AllocationConfigurationException { if (allocFile == null) { return; } LOG.info("Loading allocation file " + allocFile); // Create some temporary hashmaps to hold the new allocs, and we only save // them in our fields if we have parsed the entire allocs file successfully. Map<String, Resource> minQueueResources = new HashMap<String, Resource>(); Map<String, Resource> maxQueueResources = new HashMap<String, Resource>(); Map<String, Integer> queueMaxApps = new HashMap<String, Integer>(); Map<String, Integer> userMaxApps = new HashMap<String, Integer>(); Map<String, Float> queueMaxAMShares = new HashMap<String, Float>(); Map<String, ResourceWeights> queueWeights = new HashMap<String, ResourceWeights>(); Map<String, SchedulingPolicy> queuePolicies = new HashMap<String, SchedulingPolicy>(); Map<String, Long> minSharePreemptionTimeouts = new HashMap<String, Long>(); Map<String, Long> fairSharePreemptionTimeouts = new HashMap<String, Long>(); Map<String, Float> fairSharePreemptionThresholds = new HashMap<String, Float>(); Map<String, Map<QueueACL, AccessControlList>> queueAcls = new HashMap<String, Map<QueueACL, AccessControlList>>(); Map<String, Map<ReservationACL, AccessControlList>> reservationAcls = new HashMap<String, Map<ReservationACL, AccessControlList>>(); Set<String> reservableQueues = new HashSet<String>(); int userMaxAppsDefault = Integer.MAX_VALUE; int queueMaxAppsDefault = Integer.MAX_VALUE; Resource queueMaxResourcesDefault = Resources.unbounded(); float queueMaxAMShareDefault = 0.5f; long defaultFairSharePreemptionTimeout = Long.MAX_VALUE; long defaultMinSharePreemptionTimeout = Long.MAX_VALUE; float defaultFairSharePreemptionThreshold = 0.5f; SchedulingPolicy defaultSchedPolicy = SchedulingPolicy.DEFAULT_POLICY; // Reservation global configuration knobs String planner = null; String reservationAgent = null; String reservationAdmissionPolicy = null; QueuePlacementPolicy newPlacementPolicy = null; // Remember all queue names so we can display them on web UI, etc. // configuredQueues is segregated based on whether it is a leaf queue // or a parent queue. This information is used for creating queues // and also for making queue placement decisions(QueuePlacementRule.java). Map<FSQueueType, Set<String>> configuredQueues = new HashMap<FSQueueType, Set<String>>(); for (FSQueueType queueType : FSQueueType.values()) { configuredQueues.put(queueType, new HashSet<String>()); } // Read and parse the allocations file. DocumentBuilderFactory docBuilderFactory = DocumentBuilderFactory.newInstance(); docBuilderFactory.setIgnoringComments(true); DocumentBuilder builder = docBuilderFactory.newDocumentBuilder(); Document doc = builder.parse(allocFile); Element root = doc.getDocumentElement(); if (!"allocations".equals(root.getTagName())) throw new AllocationConfigurationException( "Bad fair scheduler config " + "file: top-level element not <allocations>"); NodeList elements = root.getChildNodes(); List<Element> queueElements = new ArrayList<Element>(); Element placementPolicyElement = null; for (int i = 0; i < elements.getLength(); i++) { Node node = elements.item(i); if (node instanceof Element) { Element element = (Element) node; if ("queue".equals(element.getTagName()) || "pool".equals(element.getTagName())) { queueElements.add(element); } else if ("user".equals(element.getTagName())) { String userName = element.getAttribute("name"); NodeList fields = element.getChildNodes(); for (int j = 0; j < fields.getLength(); j++) { Node fieldNode = fields.item(j); if (!(fieldNode instanceof Element)) continue; Element field = (Element) fieldNode; if ("maxRunningApps".equals(field.getTagName())) { String text = ((Text) field.getFirstChild()).getData().trim(); int val = Integer.parseInt(text); userMaxApps.put(userName, val); } } } else if ("queueMaxResourcesDefault".equals(element.getTagName())) { String text = ((Text) element.getFirstChild()).getData().trim(); Resource val = FairSchedulerConfiguration.parseResourceConfigValue(text); queueMaxResourcesDefault = val; } else if ("userMaxAppsDefault".equals(element.getTagName())) { String text = ((Text) element.getFirstChild()).getData().trim(); int val = Integer.parseInt(text); userMaxAppsDefault = val; } else if ("defaultFairSharePreemptionTimeout".equals(element.getTagName())) { String text = ((Text) element.getFirstChild()).getData().trim(); long val = Long.parseLong(text) * 1000L; defaultFairSharePreemptionTimeout = val; } else if ("fairSharePreemptionTimeout".equals(element.getTagName())) { if (defaultFairSharePreemptionTimeout == Long.MAX_VALUE) { String text = ((Text) element.getFirstChild()).getData().trim(); long val = Long.parseLong(text) * 1000L; defaultFairSharePreemptionTimeout = val; } } else if ("defaultMinSharePreemptionTimeout".equals(element.getTagName())) { String text = ((Text) element.getFirstChild()).getData().trim(); long val = Long.parseLong(text) * 1000L; defaultMinSharePreemptionTimeout = val; } else if ("defaultFairSharePreemptionThreshold".equals(element.getTagName())) { String text = ((Text) element.getFirstChild()).getData().trim(); float val = Float.parseFloat(text); val = Math.max(Math.min(val, 1.0f), 0.0f); defaultFairSharePreemptionThreshold = val; } else if ("queueMaxAppsDefault".equals(element.getTagName())) { String text = ((Text) element.getFirstChild()).getData().trim(); int val = Integer.parseInt(text); queueMaxAppsDefault = val; } else if ("queueMaxAMShareDefault".equals(element.getTagName())) { String text = ((Text) element.getFirstChild()).getData().trim(); float val = Float.parseFloat(text); val = Math.min(val, 1.0f); queueMaxAMShareDefault = val; } else if ("defaultQueueSchedulingPolicy".equals(element.getTagName()) || "defaultQueueSchedulingMode".equals(element.getTagName())) { String text = ((Text) element.getFirstChild()).getData().trim(); defaultSchedPolicy = SchedulingPolicy.parse(text); } else if ("queuePlacementPolicy".equals(element.getTagName())) { placementPolicyElement = element; } else if ("reservation-planner".equals(element.getTagName())) { String text = ((Text) element.getFirstChild()).getData().trim(); planner = text; } else if ("reservation-agent".equals(element.getTagName())) { String text = ((Text) element.getFirstChild()).getData().trim(); reservationAgent = text; } else if ("reservation-policy".equals(element.getTagName())) { String text = ((Text) element.getFirstChild()).getData().trim(); reservationAdmissionPolicy = text; } else { LOG.warn("Bad element in allocations file: " + element.getTagName()); } } } // Load queue elements. A root queue can either be included or omitted. If // it's included, all other queues must be inside it. for (Element element : queueElements) { String parent = "root"; if (element.getAttribute("name").equalsIgnoreCase("root")) { if (queueElements.size() > 1) { throw new AllocationConfigurationException( "If configuring root queue," + " no other queues can be placed alongside it."); } parent = null; } loadQueue(parent, element, minQueueResources, maxQueueResources, queueMaxApps, userMaxApps, queueMaxAMShares, queueWeights, queuePolicies, minSharePreemptionTimeouts, fairSharePreemptionTimeouts, fairSharePreemptionThresholds, queueAcls, reservationAcls, configuredQueues, reservableQueues); } // Load placement policy and pass it configured queues Configuration conf = getConfig(); if (placementPolicyElement != null) { newPlacementPolicy = QueuePlacementPolicy.fromXml(placementPolicyElement, configuredQueues, conf); } else { newPlacementPolicy = QueuePlacementPolicy.fromConfiguration(conf, configuredQueues); } // Set the min/fair share preemption timeout for the root queue if (!minSharePreemptionTimeouts.containsKey(QueueManager.ROOT_QUEUE)) { minSharePreemptionTimeouts.put(QueueManager.ROOT_QUEUE, defaultMinSharePreemptionTimeout); } if (!fairSharePreemptionTimeouts.containsKey(QueueManager.ROOT_QUEUE)) { fairSharePreemptionTimeouts.put(QueueManager.ROOT_QUEUE, defaultFairSharePreemptionTimeout); } // Set the fair share preemption threshold for the root queue if (!fairSharePreemptionThresholds.containsKey(QueueManager.ROOT_QUEUE)) { fairSharePreemptionThresholds.put(QueueManager.ROOT_QUEUE, defaultFairSharePreemptionThreshold); } ReservationQueueConfiguration globalReservationQueueConfig = new ReservationQueueConfiguration(); if (planner != null) { globalReservationQueueConfig.setPlanner(planner); } if (reservationAdmissionPolicy != null) { globalReservationQueueConfig.setReservationAdmissionPolicy(reservationAdmissionPolicy); } if (reservationAgent != null) { globalReservationQueueConfig.setReservationAgent(reservationAgent); } AllocationConfiguration info = new AllocationConfiguration(minQueueResources, maxQueueResources, queueMaxApps, userMaxApps, queueWeights, queueMaxAMShares, userMaxAppsDefault, queueMaxAppsDefault, queueMaxResourcesDefault, queueMaxAMShareDefault, queuePolicies, defaultSchedPolicy, minSharePreemptionTimeouts, fairSharePreemptionTimeouts, fairSharePreemptionThresholds, queueAcls, reservationAcls, newPlacementPolicy, configuredQueues, globalReservationQueueConfig, reservableQueues); lastSuccessfulReload = clock.getTime(); lastReloadAttemptFailed = false; reloadListener.onReload(info); }
From source file:org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.AllocationFileLoaderService.java
/** * Loads a queue from a queue element in the configuration file */// w w w. jav a 2s . c o m private void loadQueue(String parentName, Element element, Map<String, Resource> minQueueResources, Map<String, Resource> maxQueueResources, Map<String, Integer> queueMaxApps, Map<String, Integer> userMaxApps, Map<String, Float> queueMaxAMShares, Map<String, ResourceWeights> queueWeights, Map<String, SchedulingPolicy> queuePolicies, Map<String, Long> minSharePreemptionTimeouts, Map<String, Long> fairSharePreemptionTimeouts, Map<String, Float> fairSharePreemptionThresholds, Map<String, Map<QueueACL, AccessControlList>> queueAcls, Map<String, Map<ReservationACL, AccessControlList>> resAcls, Map<FSQueueType, Set<String>> configuredQueues, Set<String> reservableQueues) throws AllocationConfigurationException { String queueName = element.getAttribute("name").trim(); if (queueName.contains(".")) { throw new AllocationConfigurationException("Bad fair scheduler config " + "file: queue name (" + queueName + ") shouldn't contain period."); } if (queueName.isEmpty()) { throw new AllocationConfigurationException("Bad fair scheduler config " + "file: queue name shouldn't be empty or " + "consist only of whitespace."); } if (parentName != null) { queueName = parentName + "." + queueName; } Map<QueueACL, AccessControlList> acls = new HashMap<QueueACL, AccessControlList>(); Map<ReservationACL, AccessControlList> racls = new HashMap<>(); NodeList fields = element.getChildNodes(); boolean isLeaf = true; for (int j = 0; j < fields.getLength(); j++) { Node fieldNode = fields.item(j); if (!(fieldNode instanceof Element)) continue; Element field = (Element) fieldNode; if ("minResources".equals(field.getTagName())) { String text = ((Text) field.getFirstChild()).getData().trim(); Resource val = FairSchedulerConfiguration.parseResourceConfigValue(text); minQueueResources.put(queueName, val); } else if ("maxResources".equals(field.getTagName())) { String text = ((Text) field.getFirstChild()).getData().trim(); Resource val = FairSchedulerConfiguration.parseResourceConfigValue(text); maxQueueResources.put(queueName, val); } else if ("maxRunningApps".equals(field.getTagName())) { String text = ((Text) field.getFirstChild()).getData().trim(); int val = Integer.parseInt(text); queueMaxApps.put(queueName, val); } else if ("maxAMShare".equals(field.getTagName())) { String text = ((Text) field.getFirstChild()).getData().trim(); float val = Float.parseFloat(text); val = Math.min(val, 1.0f); queueMaxAMShares.put(queueName, val); } else if ("weight".equals(field.getTagName())) { String text = ((Text) field.getFirstChild()).getData().trim(); double val = Double.parseDouble(text); queueWeights.put(queueName, new ResourceWeights((float) val)); } else if ("minSharePreemptionTimeout".equals(field.getTagName())) { String text = ((Text) field.getFirstChild()).getData().trim(); long val = Long.parseLong(text) * 1000L; minSharePreemptionTimeouts.put(queueName, val); } else if ("fairSharePreemptionTimeout".equals(field.getTagName())) { String text = ((Text) field.getFirstChild()).getData().trim(); long val = Long.parseLong(text) * 1000L; fairSharePreemptionTimeouts.put(queueName, val); } else if ("fairSharePreemptionThreshold".equals(field.getTagName())) { String text = ((Text) field.getFirstChild()).getData().trim(); float val = Float.parseFloat(text); val = Math.max(Math.min(val, 1.0f), 0.0f); fairSharePreemptionThresholds.put(queueName, val); } else if ("schedulingPolicy".equals(field.getTagName()) || "schedulingMode".equals(field.getTagName())) { String text = ((Text) field.getFirstChild()).getData().trim(); SchedulingPolicy policy = SchedulingPolicy.parse(text); queuePolicies.put(queueName, policy); } else if ("aclSubmitApps".equals(field.getTagName())) { String text = ((Text) field.getFirstChild()).getData(); acls.put(QueueACL.SUBMIT_APPLICATIONS, new AccessControlList(text)); } else if ("aclAdministerApps".equals(field.getTagName())) { String text = ((Text) field.getFirstChild()).getData(); acls.put(QueueACL.ADMINISTER_QUEUE, new AccessControlList(text)); } else if ("aclAdministerReservations".equals(field.getTagName())) { String text = ((Text) field.getFirstChild()).getData(); racls.put(ReservationACL.ADMINISTER_RESERVATIONS, new AccessControlList(text)); } else if ("aclListReservations".equals(field.getTagName())) { String text = ((Text) field.getFirstChild()).getData(); racls.put(ReservationACL.LIST_RESERVATIONS, new AccessControlList(text)); } else if ("aclSubmitReservations".equals(field.getTagName())) { String text = ((Text) field.getFirstChild()).getData(); racls.put(ReservationACL.SUBMIT_RESERVATIONS, new AccessControlList(text)); } else if ("reservation".equals(field.getTagName())) { isLeaf = false; reservableQueues.add(queueName); configuredQueues.get(FSQueueType.PARENT).add(queueName); } else if ("queue".endsWith(field.getTagName()) || "pool".equals(field.getTagName())) { loadQueue(queueName, field, minQueueResources, maxQueueResources, queueMaxApps, userMaxApps, queueMaxAMShares, queueWeights, queuePolicies, minSharePreemptionTimeouts, fairSharePreemptionTimeouts, fairSharePreemptionThresholds, queueAcls, resAcls, configuredQueues, reservableQueues); isLeaf = false; } } if (isLeaf) { // if a leaf in the alloc file is marked as type='parent' // then store it under 'parent' if ("parent".equals(element.getAttribute("type"))) { configuredQueues.get(FSQueueType.PARENT).add(queueName); } else { configuredQueues.get(FSQueueType.LEAF).add(queueName); } } else { if ("parent".equals(element.getAttribute("type"))) { throw new AllocationConfigurationException("Both <reservation> and " + "type=\"parent\" found for queue " + queueName + " which is " + "unsupported"); } configuredQueues.get(FSQueueType.PARENT).add(queueName); } queueAcls.put(queueName, acls); resAcls.put(queueName, racls); if (maxQueueResources.containsKey(queueName) && minQueueResources.containsKey(queueName) && !Resources.fitsIn(minQueueResources.get(queueName), maxQueueResources.get(queueName))) { LOG.warn(String.format("Queue %s has max resources %s less than min resources %s", queueName, maxQueueResources.get(queueName), minQueueResources.get(queueName))); } }
From source file:org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.PoolManager.java
/** * Updates the allocation list from the allocation config file. This file is * expected to be in the XML format specified in the design doc. * //from w ww . j a v a 2 s . com * @throws IOException if the config file cannot be read. * @throws AllocationConfigurationException if allocations are invalid. * @throws ParserConfigurationException if XML parser is misconfigured. * @throws SAXException if config file is malformed. */ public void reloadAllocs() throws IOException, ParserConfigurationException, SAXException, AllocationConfigurationException { if (allocFile == null) return; // Create some temporary hashmaps to hold the new allocs, and we only save // them in our fields if we have parsed the entire allocs file successfully. Map<String, Resource> minPoolResources = new HashMap<String, Resource>(); Map<String, Resource> maxPoolResources = new HashMap<String, Resource>(); Map<String, Integer> poolMaxApps = new HashMap<String, Integer>(); Map<String, Integer> userMaxApps = new HashMap<String, Integer>(); Map<String, Double> poolWeights = new HashMap<String, Double>(); Map<String, SchedulingMode> poolModes = new HashMap<String, SchedulingMode>(); Map<String, Long> minSharePreemptionTimeouts = new HashMap<String, Long>(); int userMaxAppsDefault = Integer.MAX_VALUE; int poolMaxAppsDefault = Integer.MAX_VALUE; long fairSharePreemptionTimeout = Long.MAX_VALUE; long defaultMinSharePreemptionTimeout = Long.MAX_VALUE; SchedulingMode defaultSchedulingMode = SchedulingMode.FAIR; // Remember all pool names so we can display them on web UI, etc. List<String> poolNamesInAllocFile = new ArrayList<String>(); // Read and parse the allocations file. DocumentBuilderFactory docBuilderFactory = DocumentBuilderFactory.newInstance(); docBuilderFactory.setIgnoringComments(true); DocumentBuilder builder = docBuilderFactory.newDocumentBuilder(); Document doc; if (allocFile instanceof String) { doc = builder.parse(new File((String) allocFile)); } else { doc = builder.parse(allocFile.toString()); } Element root = doc.getDocumentElement(); if (!"allocations".equals(root.getTagName())) throw new AllocationConfigurationException( "Bad fair scheduler config " + "file: top-level element not <allocations>"); NodeList elements = root.getChildNodes(); for (int i = 0; i < elements.getLength(); i++) { Node node = elements.item(i); if (!(node instanceof Element)) continue; Element element = (Element) node; if ("pool".equals(element.getTagName())) { String poolName = element.getAttribute("name"); poolNamesInAllocFile.add(poolName); NodeList fields = element.getChildNodes(); for (int j = 0; j < fields.getLength(); j++) { Node fieldNode = fields.item(j); if (!(fieldNode instanceof Element)) continue; Element field = (Element) fieldNode; if ("minResources".equals(field.getTagName())) { String text = ((Text) field.getFirstChild()).getData().trim(); int val = Integer.parseInt(text); minPoolResources.put(poolName, Resources.createResource(val)); } else if ("maxResources".equals(field.getTagName())) { String text = ((Text) field.getFirstChild()).getData().trim(); int val = Integer.parseInt(text); maxPoolResources.put(poolName, Resources.createResource(val)); } else if ("maxRunningApps".equals(field.getTagName())) { String text = ((Text) field.getFirstChild()).getData().trim(); int val = Integer.parseInt(text); poolMaxApps.put(poolName, val); } else if ("weight".equals(field.getTagName())) { String text = ((Text) field.getFirstChild()).getData().trim(); double val = Double.parseDouble(text); poolWeights.put(poolName, val); } else if ("minSharePreemptionTimeout".equals(field.getTagName())) { String text = ((Text) field.getFirstChild()).getData().trim(); long val = Long.parseLong(text) * 1000L; minSharePreemptionTimeouts.put(poolName, val); } else if ("schedulingMode".equals(field.getTagName())) { String text = ((Text) field.getFirstChild()).getData().trim(); poolModes.put(poolName, parseSchedulingMode(text)); } } if (maxPoolResources.containsKey(poolName) && minPoolResources.containsKey(poolName) && Resources.lessThan(maxPoolResources.get(poolName), minPoolResources.get(poolName))) { LOG.warn(String.format("Pool %s has max resources %d less than min resources %d", poolName, maxPoolResources.get(poolName), minPoolResources.get(poolName))); } } else if ("user".equals(element.getTagName())) { String userName = element.getAttribute("name"); NodeList fields = element.getChildNodes(); for (int j = 0; j < fields.getLength(); j++) { Node fieldNode = fields.item(j); if (!(fieldNode instanceof Element)) continue; Element field = (Element) fieldNode; if ("maxRunningApps".equals(field.getTagName())) { String text = ((Text) field.getFirstChild()).getData().trim(); int val = Integer.parseInt(text); userMaxApps.put(userName, val); } } } else if ("userMaxAppsDefault".equals(element.getTagName())) { String text = ((Text) element.getFirstChild()).getData().trim(); int val = Integer.parseInt(text); userMaxAppsDefault = val; } else if ("poolMaxAppsDefault".equals(element.getTagName())) { String text = ((Text) element.getFirstChild()).getData().trim(); int val = Integer.parseInt(text); poolMaxAppsDefault = val; } else if ("fairSharePreemptionTimeout".equals(element.getTagName())) { String text = ((Text) element.getFirstChild()).getData().trim(); long val = Long.parseLong(text) * 1000L; fairSharePreemptionTimeout = val; } else if ("defaultMinSharePreemptionTimeout".equals(element.getTagName())) { String text = ((Text) element.getFirstChild()).getData().trim(); long val = Long.parseLong(text) * 1000L; defaultMinSharePreemptionTimeout = val; } else if ("defaultPoolSchedulingMode".equals(element.getTagName())) { String text = ((Text) element.getFirstChild()).getData().trim(); defaultSchedulingMode = parseSchedulingMode(text); } else { LOG.warn("Bad element in allocations file: " + element.getTagName()); } } // Commit the reload; also create any pool defined in the alloc file // if it does not already exist, so it can be displayed on the web UI. synchronized (this) { this.minPoolResources = minPoolResources; this.maxPoolResources = maxPoolResources; this.poolMaxApps = poolMaxApps; this.userMaxApps = userMaxApps; this.poolWeights = poolWeights; this.minSharePreemptionTimeouts = minSharePreemptionTimeouts; this.userMaxAppsDefault = userMaxAppsDefault; this.poolMaxAppsDefault = poolMaxAppsDefault; this.fairSharePreemptionTimeout = fairSharePreemptionTimeout; this.defaultMinSharePreemptionTimeout = defaultMinSharePreemptionTimeout; this.defaultSchedulingMode = defaultSchedulingMode; for (String name : poolNamesInAllocFile) { Pool pool = getPool(name); if (poolModes.containsKey(name)) { pool.setSchedulingMode(poolModes.get(name)); } else { pool.setSchedulingMode(defaultSchedulingMode); } } } }