Java tutorial
/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ package org.apache.hive.ptest.execution; import java.io.File; import java.io.IOException; import java.util.Collection; import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Set; import java.util.concurrent.BlockingQueue; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.TimeUnit; import com.google.common.base.Joiner; import org.apache.hive.ptest.execution.conf.Host; import org.apache.hive.ptest.execution.conf.QFileTestBatch; import org.apache.hive.ptest.execution.conf.TestBatch; import org.apache.hive.ptest.execution.context.ExecutionContext; import org.slf4j.Logger; import com.google.common.base.Supplier; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; import com.google.common.collect.Sets; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; public class ExecutionPhase extends Phase { private static final long FOUR_HOURS = 4L * 60L * 60L * 1000L; private final ExecutionContext executionContext; private final HostExecutorBuilder hostExecutorBuilder; private final File succeededLogDir; private final File failedLogDir; private final BlockingQueue<TestBatch> parallelWorkQueue; private final BlockingQueue<TestBatch> isolatedWorkQueue; private final Set<String> executedTests; private final Set<String> failedTests; private final Supplier<List<TestBatch>> testBatchSupplier; private final Set<TestBatch> failedTestResults; public ExecutionPhase(List<HostExecutor> hostExecutors, ExecutionContext executionContext, HostExecutorBuilder hostExecutorBuilder, LocalCommandFactory localCommandFactory, ImmutableMap<String, String> templateDefaults, File succeededLogDir, File failedLogDir, Supplier<List<TestBatch>> testBatchSupplier, Set<String> executedTests, Set<String> failedTests, Logger logger) throws IOException { super(hostExecutors, localCommandFactory, templateDefaults, logger); this.executionContext = executionContext; this.hostExecutorBuilder = hostExecutorBuilder; this.succeededLogDir = succeededLogDir; this.failedLogDir = failedLogDir; this.testBatchSupplier = testBatchSupplier; this.executedTests = executedTests; this.failedTests = failedTests; this.parallelWorkQueue = new LinkedBlockingQueue<TestBatch>(); this.isolatedWorkQueue = new LinkedBlockingQueue<TestBatch>(); this.failedTestResults = Collections.synchronizedSet(new HashSet<TestBatch>()); } @Override public void execute() throws Throwable { long start = System.currentTimeMillis(); List<TestBatch> testBatches = Lists.newArrayList(); for (TestBatch batch : testBatchSupplier.get()) { testBatches.add(batch); if (batch.isParallel()) { parallelWorkQueue.add(batch); } else { isolatedWorkQueue.add(batch); } } logger.info("ParallelWorkQueueSize={}, IsolatedWorkQueueSize={}", parallelWorkQueue.size(), isolatedWorkQueue.size()); if (logger.isDebugEnabled()) { for (TestBatch testBatch : parallelWorkQueue) { logger.debug("PBatch: {}", testBatch); } for (TestBatch testBatch : isolatedWorkQueue) { logger.debug("IBatch: {}", testBatch); } } try { int expectedNumHosts = hostExecutors.size(); initalizeHosts(); do { replaceBadHosts(expectedNumHosts); List<ListenableFuture<Void>> results = Lists.newArrayList(); for (HostExecutor hostExecutor : ImmutableList.copyOf(hostExecutors)) { results.add(hostExecutor.submitTests(parallelWorkQueue, isolatedWorkQueue, failedTestResults)); } Futures.allAsList(results).get(); } while (!(parallelWorkQueue.isEmpty() && isolatedWorkQueue.isEmpty())); for (TestBatch batch : testBatches) { File batchLogDir; if (failedTestResults.contains(batch)) { batchLogDir = new File(failedLogDir, batch.getName()); } else { batchLogDir = new File(succeededLogDir, batch.getName()); } JUnitReportParser parser = new JUnitReportParser(logger, batchLogDir); executedTests.addAll(parser.getAllExecutedTests()); for (String failedTest : parser.getAllFailedTests()) { failedTests.add(failedTest + " (batchId=" + batch.getBatchId() + ")"); } // if the TEST*.xml was not generated or was corrupt, let someone know if (parser.getTestClassesWithReportAvailable().size() < batch.getTestClasses().size()) { Set<String> expTestClasses = new HashSet<>(batch.getTestClasses()); expTestClasses.removeAll(parser.getTestClassesWithReportAvailable()); for (String testClass : expTestClasses) { StringBuilder messageBuilder = new StringBuilder(); messageBuilder.append(testClass) .append(" - did not produce a TEST-*.xml file (likely timed out)") .append(" (batchId=").append(batch.getBatchId()).append(")"); if (batch instanceof QFileTestBatch) { Collection<String> tests = ((QFileTestBatch) batch).getTests(); if (tests.size() != 0) { messageBuilder.append("\n\t["); messageBuilder.append(Joiner.on(",").join(tests)); messageBuilder.append("]"); } } failedTests.add(messageBuilder.toString()); } } } } finally { long elapsed = System.currentTimeMillis() - start; logger.info( "PERF: exec phase " + TimeUnit.MINUTES.convert(elapsed, TimeUnit.MILLISECONDS) + " minutes"); } } private void replaceBadHosts(int expectedNumHosts) throws Exception { Set<Host> goodHosts = Sets.newHashSet(); for (HostExecutor hostExecutor : ImmutableList.copyOf(hostExecutors)) { if (hostExecutor.isBad()) { logger.info("Removing host during execution phase: " + hostExecutor.getHost()); executionContext.addBadHost(hostExecutor.getHost()); hostExecutors.remove(hostExecutor); } else { goodHosts.add(hostExecutor.getHost()); } } long start = System.currentTimeMillis(); while (hostExecutors.size() < expectedNumHosts) { if (System.currentTimeMillis() - start > FOUR_HOURS) { throw new RuntimeException("Waited over fours for hosts, still have only " + hostExecutors.size() + " hosts out of an expected " + expectedNumHosts); } logger.warn("Only " + hostExecutors.size() + " hosts out of an expected " + expectedNumHosts + ", attempting to replace bad hosts"); TimeUnit.MINUTES.sleep(1); executionContext.replaceBadHosts(); for (Host host : executionContext.getHosts()) { if (!goodHosts.contains(host)) { HostExecutor hostExecutor = hostExecutorBuilder.build(host); initalizeHost(hostExecutor); if (hostExecutor.isBad()) { executionContext.addBadHost(hostExecutor.getHost()); } else { logger.info("Adding new host during execution phase: " + host); hostExecutors.add(hostExecutor); } } } } } }