List of usage examples for org.apache.commons.lang ArrayUtils addAll
public static double[] addAll(double[] array1, double[] array2)
Adds all the elements of the given arrays into a new array.
From source file:org.apache.hadoop.mapreduce.v2.TestMRJobs.java
@Test(timeout = 120000) public void testThreadDumpOnTaskTimeout() throws IOException, InterruptedException, ClassNotFoundException { if (!(new File(MiniMRYarnCluster.APPJAR)).exists()) { LOG.info("MRAppJar " + MiniMRYarnCluster.APPJAR + " not found. Not running test."); return;//from w ww . ja v a 2 s. co m } final SleepJob sleepJob = new SleepJob(); final JobConf sleepConf = new JobConf(mrCluster.getConfig()); sleepConf.setLong(MRJobConfig.TASK_TIMEOUT, 3 * 1000L); sleepConf.setInt(MRJobConfig.MAP_MAX_ATTEMPTS, 1); sleepJob.setConf(sleepConf); if (this instanceof TestUberAM) { sleepConf.setInt(MRJobConfig.MR_AM_TO_RM_HEARTBEAT_INTERVAL_MS, 30 * 1000); } // sleep for 10 seconds to trigger a kill with thread dump final Job job = sleepJob.createJob(1, 0, 10 * 60 * 1000L, 1, 0L, 0); job.setJarByClass(SleepJob.class); job.addFileToClassPath(APP_JAR); // The AppMaster jar itself. job.waitForCompletion(true); final JobId jobId = TypeConverter.toYarn(job.getJobID()); final ApplicationId appID = jobId.getAppId(); int pollElapsed = 0; while (true) { Thread.sleep(1000); pollElapsed += 1000; if (TERMINAL_RM_APP_STATES .contains(mrCluster.getResourceManager().getRMContext().getRMApps().get(appID).getState())) { break; } if (pollElapsed >= 60000) { LOG.warn("application did not reach terminal state within 60 seconds"); break; } } // Job finished, verify logs // final String appIdStr = appID.toString(); final String appIdSuffix = appIdStr.substring("application_".length(), appIdStr.length()); final String containerGlob = "container_" + appIdSuffix + "_*_*"; final String syslogGlob = appIdStr + Path.SEPARATOR + containerGlob + Path.SEPARATOR + TaskLog.LogName.SYSLOG; int numAppMasters = 0; int numMapTasks = 0; String user = UserGroupInformation.getCurrentUser().getUserName(); String userFolder; try { MessageDigest digest = MessageDigest .getInstance(mrCluster.getResourceManager().getRMContext().getUserFolderHashAlgo()); byte[] userBytes = user.getBytes(StandardCharsets.UTF_8); byte[] hashBase = ArrayUtils.addAll(userBytes, mrCluster.getResourceManager().getRMContext().getSeed()); byte[] hash = digest.digest(hashBase); userFolder = Base64.encodeBase64URLSafeString(hash); } catch (NoSuchAlgorithmException ex) { LOG.error("error while creating userFolder random string", ex); throw new Error("error while creating userFolder random string", ex); } for (int i = 0; i < NUM_NODE_MGRS; i++) { final Configuration nmConf = mrCluster.getNodeManager(i).getConfig(); for (String logDir : nmConf.getTrimmedStrings(YarnConfiguration.NM_LOG_DIRS)) { Path userLogFolder = new Path(logDir, userFolder); final Path absSyslogGlob = new Path(userLogFolder + Path.SEPARATOR + syslogGlob); LOG.info("Checking for glob: " + absSyslogGlob); for (FileStatus syslog : localFs.globStatus(absSyslogGlob)) { boolean foundAppMaster = false; boolean foundThreadDump = false; // Determine the container type final BufferedReader syslogReader = new BufferedReader( new InputStreamReader(localFs.open(syslog.getPath()))); try { for (String line; (line = syslogReader.readLine()) != null;) { if (line.contains(MRAppMaster.class.getName())) { foundAppMaster = true; break; } } } finally { syslogReader.close(); } // Check for thread dump in stdout final Path stdoutPath = new Path(syslog.getPath().getParent(), TaskLog.LogName.STDOUT.toString()); final BufferedReader stdoutReader = new BufferedReader( new InputStreamReader(localFs.open(stdoutPath))); try { for (String line; (line = stdoutReader.readLine()) != null;) { if (line.contains("Full thread dump")) { foundThreadDump = true; break; } } } finally { stdoutReader.close(); } if (foundAppMaster) { numAppMasters++; if (this instanceof TestUberAM) { Assert.assertTrue("No thread dump", foundThreadDump); } else { Assert.assertFalse("Unexpected thread dump", foundThreadDump); } } else { numMapTasks++; Assert.assertTrue("No thread dump", foundThreadDump); } } } } // Make sure we checked non-empty set // Assert.assertEquals("No AppMaster log found!", 1, numAppMasters); if (sleepConf.getBoolean(MRJobConfig.JOB_UBERTASK_ENABLE, false)) { Assert.assertSame("MapTask log with uber found!", 0, numMapTasks); } else { Assert.assertSame("No MapTask log found!", 1, numMapTasks); } }
From source file:org.apache.hadoop.mapreduce.v2.TestMRJobsWithProfiler.java
private void testProfilerInternal(boolean useDefault) throws Exception { if (!(new File(MiniMRYarnCluster.APPJAR)).exists()) { LOG.info("MRAppJar " + MiniMRYarnCluster.APPJAR + " not found. Not running test."); return;// ww w.ja v a 2s . co m } final SleepJob sleepJob = new SleepJob(); final JobConf sleepConf = new JobConf(mrCluster.getConfig()); sleepConf.setProfileEnabled(true); sleepConf.setProfileTaskRange(true, String.valueOf(PROFILED_TASK_ID)); sleepConf.setProfileTaskRange(false, String.valueOf(PROFILED_TASK_ID)); if (!useDefault) { // use hprof for map to profile.out sleepConf.set(MRJobConfig.TASK_MAP_PROFILE_PARAMS, "-agentlib:hprof=cpu=samples,heap=sites,force=n,thread=y,verbose=n," + "file=%s"); // use Xprof for reduce to stdout sleepConf.set(MRJobConfig.TASK_REDUCE_PROFILE_PARAMS, "-Xprof"); } sleepJob.setConf(sleepConf); // 2-map-2-reduce SleepJob final Job job = sleepJob.createJob(2, 2, 500, 1, 500, 1); job.setJarByClass(SleepJob.class); job.addFileToClassPath(APP_JAR); // The AppMaster jar itself. job.waitForCompletion(true); final JobId jobId = TypeConverter.toYarn(job.getJobID()); final ApplicationId appID = jobId.getAppId(); int pollElapsed = 0; while (true) { Thread.sleep(1000); pollElapsed += 1000; if (TERMINAL_RM_APP_STATES .contains(mrCluster.getResourceManager().getRMContext().getRMApps().get(appID).getState())) { break; } if (pollElapsed >= 60000) { LOG.warn("application did not reach terminal state within 60 seconds"); break; } } Assert.assertEquals(RMAppState.FINISHED, mrCluster.getResourceManager().getRMContext().getRMApps().get(appID).getState()); // Job finished, verify logs // final Configuration nmConf = mrCluster.getNodeManager(0).getConfig(); final String appIdStr = appID.toString(); final String appIdSuffix = appIdStr.substring("application_".length(), appIdStr.length()); final String containerGlob = "container_" + appIdSuffix + "_*_*"; final Map<TaskAttemptID, Path> taLogDirs = new HashMap<TaskAttemptID, Path>(); final Pattern taskPattern = Pattern .compile(".*Task:(attempt_" + appIdSuffix + "_[rm]_" + "[0-9]+_[0-9]+).*"); String user = UserGroupInformation.getCurrentUser().getUserName(); String userFolder; try { MessageDigest digest = MessageDigest .getInstance(mrCluster.getResourceManager().getRMContext().getUserFolderHashAlgo()); byte[] userBytes = user.getBytes(StandardCharsets.UTF_8); byte[] hashBase = ArrayUtils.addAll(userBytes, mrCluster.getResourceManager().getRMContext().getSeed()); byte[] hash = digest.digest(hashBase); userFolder = org.apache.commons.codec.binary.Base64.encodeBase64URLSafeString(hash); } catch (NoSuchAlgorithmException ex) { LOG.error("error while creating userFolder random string", ex); throw new Error("error while creating userFolder random string", ex); } for (String logDir : nmConf.getTrimmedStrings(YarnConfiguration.NM_LOG_DIRS)) { // filter out MRAppMaster and create attemptId->logDir map // for (FileStatus fileStatus : localFs .globStatus(new Path(logDir + Path.SEPARATOR + userFolder + Path.SEPARATOR + appIdStr + Path.SEPARATOR + containerGlob + Path.SEPARATOR + TaskLog.LogName.SYSLOG))) { final BufferedReader br = new BufferedReader( new InputStreamReader(localFs.open(fileStatus.getPath()))); String line; while ((line = br.readLine()) != null) { final Matcher m = taskPattern.matcher(line); if (m.matches()) { // found Task done message taLogDirs.put(TaskAttemptID.forName(m.group(1)), fileStatus.getPath().getParent()); break; } } br.close(); } } Assert.assertEquals(4, taLogDirs.size()); // all 4 attempts found for (Map.Entry<TaskAttemptID, Path> dirEntry : taLogDirs.entrySet()) { final TaskAttemptID tid = dirEntry.getKey(); final Path profilePath = new Path(dirEntry.getValue(), TaskLog.LogName.PROFILE.toString()); final Path stdoutPath = new Path(dirEntry.getValue(), TaskLog.LogName.STDOUT.toString()); if (useDefault || tid.getTaskType() == TaskType.MAP) { if (tid.getTaskID().getId() == PROFILED_TASK_ID) { // verify profile.out final BufferedReader br = new BufferedReader(new InputStreamReader(localFs.open(profilePath))); final String line = br.readLine(); Assert.assertTrue("No hprof content found!", line != null && line.startsWith("JAVA PROFILE")); br.close(); Assert.assertEquals(0L, localFs.getFileStatus(stdoutPath).getLen()); } else { Assert.assertFalse("hprof file should not exist", localFs.exists(profilePath)); } } else { Assert.assertFalse("hprof file should not exist", localFs.exists(profilePath)); if (tid.getTaskID().getId() == PROFILED_TASK_ID) { // reducer is profiled with Xprof final BufferedReader br = new BufferedReader(new InputStreamReader(localFs.open(stdoutPath))); boolean flatProfFound = false; String line; while ((line = br.readLine()) != null) { if (line.startsWith("Flat profile")) { flatProfFound = true; break; } } br.close(); Assert.assertTrue("Xprof flat profile not found!", flatProfFound); } else { Assert.assertEquals(0L, localFs.getFileStatus(stdoutPath).getLen()); } } } }
From source file:org.apache.hadoop.yarn.server.resourcemanager.scheduler.AppSchedulingInfo.java
public AppSchedulingInfo(ApplicationAttemptId appAttemptId, String user, Queue queue, ActiveUsersManager activeUsersManager, long epoch, ResourceUsage appResourceUsage, RMContext rmContext) {// w w w . j a v a 2s . c om this.applicationAttemptId = appAttemptId; this.applicationId = appAttemptId.getApplicationId(); this.queue = queue; this.user = user; this.activeUsersManager = activeUsersManager; this.containerIdCounter = new AtomicLong(epoch << EPOCH_BIT_SHIFT); this.appResourceUsage = appResourceUsage; try { MessageDigest digest = MessageDigest.getInstance(rmContext.getUserFolderHashAlgo()); byte[] userBytes = user.getBytes(StandardCharsets.UTF_8); byte[] hashBase = ArrayUtils.addAll(userBytes, rmContext.getSeed()); byte[] hash = digest.digest(hashBase); userFolder = Base64.encodeBase64URLSafeString(hash); } catch (NoSuchAlgorithmException ex) { LOG.error("error while creating userFolder random string", ex); throw new Error("error while creating userFolder random string", ex); } }
From source file:org.apache.hawq.pxf.plugins.jdbc.utils.ByteUtil.java
public static byte[] mergeBytes(byte[] b1, byte[] b2) { return ArrayUtils.addAll(b1, b2); }
From source file:org.apache.james.mailbox.maildir.MaildirFolder.java
/** * Creates a map of recent messages./*from ww w . j a v a 2s. c o m*/ * * @param session * @return A {@link Map} with all uids and associated {@link MaildirMessageName}s of recent messages * @throws MailboxException If there is a problem with the uid list file */ public SortedMap<Long, MaildirMessageName> getRecentMessages(final MailboxSession session) throws MailboxException { final String[] recentFiles = getNewFolder().list(); final LinkedList<String> lines = new LinkedList<String>(); final int theLimit = recentFiles.length; return locker.executeWithLock(session, path, new LockAwareExecution<SortedMap<Long, MaildirMessageName>>() { @Override public SortedMap<Long, MaildirMessageName> execute() throws MailboxException { final SortedMap<Long, MaildirMessageName> recentMessages = new TreeMap<Long, MaildirMessageName>(); File uidList = uidFile; try { if (!uidList.isFile()) { if (!uidList.createNewFile()) throw new IOException("Could not create file " + uidList); String[] curFiles = curFolder.list(); String[] newFiles = newFolder.list(); messageCount = curFiles.length + newFiles.length; String[] allFiles = (String[]) ArrayUtils.addAll(curFiles, newFiles); for (String file : allFiles) lines.add(String.valueOf(getNextUid()) + " " + file); PrintWriter pw = new PrintWriter(uidList); try { pw.println(createUidListHeader()); for (String line : lines) pw.println(line); } finally { IOUtils.closeQuietly(pw); } } else { FileReader fileReader = null; BufferedReader reader = null; try { fileReader = new FileReader(uidList); reader = new BufferedReader(fileReader); String line = reader.readLine(); // the first line in the file contains the next uid and message count while ((line = reader.readLine()) != null) lines.add(line); } finally { IOUtils.closeQuietly(reader); IOUtils.closeQuietly(fileReader); } } int counter = 0; String line; while (counter < theLimit) { // walk backwards as recent files are supposedly recent try { line = lines.removeLast(); } catch (NoSuchElementException e) { break; // the list is empty } if (!line.equals("")) { int gap = line.indexOf(" "); if (gap == -1) { // there must be some issues in the file if no gap can be found // there must be some issues in the file if no gap can be found session.getLog() .info("Corrupted entry in uid-file " + uidList + " line " + lines.size()); continue; } Long uid = Long.valueOf(line.substring(0, gap)); String name = line.substring(gap + 1, line.length()); for (String recentFile : recentFiles) { if (recentFile.equals(name)) { recentMessages.put(uid, newMaildirMessageName(MaildirFolder.this, recentFile)); counter++; break; } } } } } catch (IOException e) { throw new MailboxException("Unable to read recent messages", e); } return recentMessages; } }, true); }
From source file:org.apache.james.mailbox.maildir.MaildirFolder.java
/** * Creates and returns a uid map (uid -> {@link MaildirMessageName}) and writes it to the disk * @return The uid map/*from ww w . java2 s . c om*/ * @throws MailboxException */ private Map<Long, MaildirMessageName> createUidFile() throws MailboxException { final Map<Long, MaildirMessageName> uidMap = new TreeMap<Long, MaildirMessageName>(); File uidList = uidFile; PrintWriter pw = null; try { if (!uidList.createNewFile()) throw new IOException("Could not create file " + uidList); lastUid = 0; String[] curFiles = curFolder.list(); String[] newFiles = newFolder.list(); messageCount = curFiles.length + newFiles.length; String[] allFiles = (String[]) ArrayUtils.addAll(curFiles, newFiles); for (String file : allFiles) uidMap.put(getNextUid(), newMaildirMessageName(MaildirFolder.this, file)); //uidMap = new TreeMap<Long, MaildirMessageName>(uidMap); pw = new PrintWriter(uidList); pw.println(createUidListHeader()); for (Entry<Long, MaildirMessageName> entry : uidMap.entrySet()) pw.println(String.valueOf(entry.getKey()) + " " + entry.getValue().getFullName()); } catch (IOException e) { throw new MailboxException("Unable to create uid file", e); } finally { IOUtils.closeQuietly(pw); } return uidMap; }
From source file:org.apache.james.mailbox.maildir.MaildirFolder.java
private Map<Long, MaildirMessageName> updateUidFile() throws MailboxException { final Map<Long, MaildirMessageName> uidMap = new TreeMap<Long, MaildirMessageName>(); File uidList = uidFile;/*from w w w . ja va 2 s .co m*/ String[] curFiles = curFolder.list(); String[] newFiles = newFolder.list(); messageCount = curFiles.length + newFiles.length; HashMap<String, Long> reverseUidMap = new HashMap<String, Long>(messageCount); FileReader fileReader = null; BufferedReader reader = null; PrintWriter pw = null; try { fileReader = new FileReader(uidList); reader = new BufferedReader(fileReader); String line = reader.readLine(); // the first line in the file contains the next uid and message count if (line != null) readUidListHeader(line); int lineNumber = 1; while ((line = reader.readLine()) != null) { if (!line.equals("")) { int gap = line.indexOf(" "); if (gap == -1) { // there must be some issues in the file if no gap can be found throw new MailboxException( "Corrupted entry in uid-file " + uidList + " line " + lineNumber++); } Long uid = Long.valueOf(line.substring(0, gap)); String name = line.substring(gap + 1, line.length()); reverseUidMap.put(stripMetaFromName(name), uid); } } String[] allFiles = (String[]) ArrayUtils.addAll(curFiles, newFiles); for (String file : allFiles) { MaildirMessageName messageName = newMaildirMessageName(MaildirFolder.this, file); Long uid = reverseUidMap.get(messageName.getBaseName()); if (uid == null) uid = getNextUid(); uidMap.put(uid, messageName); } pw = new PrintWriter(uidList); pw.println(createUidListHeader()); for (Entry<Long, MaildirMessageName> entry : uidMap.entrySet()) pw.println(String.valueOf(entry.getKey()) + " " + entry.getValue().getFullName()); } catch (IOException e) { throw new MailboxException("Unable to update uid file", e); } finally { IOUtils.closeQuietly(pw); IOUtils.closeQuietly(fileReader); IOUtils.closeQuietly(reader); } return uidMap; }
From source file:org.apache.james.mailbox.maildir.MaildirFolder.java
/** * Appends a message to the uidlist and returns its uid. * @param session//ww w .j a v a 2 s . co m * @param name The name of the message's file * @return The uid of the message * @throws IOException */ public long appendMessage(MailboxSession session, final String name) throws MailboxException { return locker.executeWithLock(session, path, new LockAwareExecution<Long>() { @Override public Long execute() throws MailboxException { File uidList = uidFile; long uid = -1; FileReader fileReader = null; BufferedReader reader = null; PrintWriter pw = null; try { if (uidList.isFile()) { fileReader = new FileReader(uidList); reader = new BufferedReader(fileReader); String line = reader.readLine(); // the first line in the file contains the next uid and message count if (line != null) readUidListHeader(line); ArrayList<String> lines = new ArrayList<String>(); while ((line = reader.readLine()) != null) lines.add(line); uid = getNextUid(); lines.add(String.valueOf(uid) + " " + name); messageCount++; pw = new PrintWriter(uidList); pw.println(createUidListHeader()); for (String entry : lines) pw.println(entry); } else { // create the file if (!uidList.createNewFile()) throw new IOException("Could not create file " + uidList); String[] curFiles = curFolder.list(); String[] newFiles = newFolder.list(); messageCount = curFiles.length + newFiles.length; ArrayList<String> lines = new ArrayList<String>(); String[] allFiles = (String[]) ArrayUtils.addAll(curFiles, newFiles); for (String file : allFiles) { long theUid = getNextUid(); lines.add(String.valueOf(theUid) + " " + file); // the listed names already include the message to append if (file.equals(name)) uid = theUid; } pw = new PrintWriter(uidList); pw.println(createUidListHeader()); for (String line : lines) pw.println(line); } } catch (IOException e) { throw new MailboxException("Unable to append msg", e); } finally { IOUtils.closeQuietly(pw); IOUtils.closeQuietly(reader); IOUtils.closeQuietly(fileReader); } if (uid == -1) { throw new MailboxException("Unable to append msg"); } else { return uid; } } }, true); }
From source file:org.apache.kylin.cube.gridtable.CuboidToGridTableMappingExt.java
@Override public DataType[] getDataTypes() { return (DataType[]) ArrayUtils.addAll(super.getDataTypes(), dynGtDataTypes.toArray(new DataType[dynGtDataTypes.size()])); }
From source file:org.apache.kylin.cube.gridtable.CuboidToGridTableMappingExt.java
@Override public ImmutableBitSet[] getColumnBlocks() { return (ImmutableBitSet[]) ArrayUtils.addAll(super.getColumnBlocks(), dynGtColBlocks.toArray(new ImmutableBitSet[dynGtColBlocks.size()])); }