Example usage for org.apache.hadoop.util StringUtils join

List of usage examples for org.apache.hadoop.util StringUtils join

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.util.*
  5. StringUtils
  6. join

Introduction

In this page you can find the example usage for org.apache.hadoop.util StringUtils join.

Prototype

public static String join(char separator, String[] strings)

Source Link

Usage

From source file:com.cloudera.kitten.lua.LuaWrapper.java

License:Open Source License

public LuaWrapper(String script, Map<String, Object> extras) {
    try {/*  w  w w .  j a va 2  s .  com*/
        this.env = JsePlatform.standardGlobals();
        LoadState.load(getClass().getResourceAsStream("/lua/kitten.lua"), "kitten.lua", env).call();
        for (Map.Entry<String, Object> e : extras.entrySet()) {
            env.set(e.getKey(), CoerceJavaToLua.coerce(e.getValue()));
        }
        InputStream luaCode = LocalDataHelper.getFileOrResource(script);
        assert luaCode != null : "Lua script " + script + " not found";
        LoadState.load(luaCode, script, env).call();
    } catch (IOException e) {
        LOG.error("Lua initialization error", e);
        throw new RuntimeException(e);
    } catch (NullPointerException e) {
        LOG.error("Lua compiler error", e);
        try {
            List<String> lines = IOUtils.readLines(LocalDataHelper.getFileOrResource(script));
            LOG.error(StringUtils.join("\n", lines));
        } catch (IOException ioe) {
        }
        throw new RuntimeException(e);
    }
}

From source file:com.kylinolap.metadata.model.cube.ParameterDesc.java

License:Apache License

public void normalizeColumnValue() {
    if (isColumnType()) {
        String values[] = value.split("\\s*,\\s*");
        for (int i = 0; i < values.length; i++)
            values[i] = values[i].toUpperCase();
        Arrays.sort(values);//from  ww w. j  av  a  2 s .co  m
        value = StringUtils.join(",", values);
    }
}

From source file:com.qubole.streamx.s3.wal.DBWAL.java

License:Apache License

@Override
public void append(String tempFile, String committedFile) throws ConnectException {
    try {/*  w w w.j a  v  a 2  s  .c  o m*/
        if (tempFile == WAL.beginMarker) {
            tempFiles.clear();
            committedFiles.clear();
        } else if (tempFile == WAL.endMarker) {
            String tempFilesCommaSeparated = StringUtils.join(",", tempFiles);
            String committedFilesCommaSeparated = StringUtils.join(",", committedFiles);

            acquireLease();
            Connection connection = cpds.getConnection();
            connection.setAutoCommit(false);

            Statement statement = connection.createStatement();
            statement.setQueryTimeout(30); // set timeout to 30 sec.

            String sql = String.format("insert into %s (tempFiles,committedFiles) values ('%s','%s')",
                    tableName, tempFilesCommaSeparated, committedFilesCommaSeparated);
            log.info("committing " + sql);
            statement.executeUpdate(sql);
            connection.commit();
        } else {
            tempFiles.add(tempFile);
            committedFiles.add(committedFile);
        }
    } catch (SQLException e) {
        log.error(e.toString());
        throw new ConnectException(e);
    }
}

From source file:com.streamsets.pipeline.stage.destination.hive.HiveTarget.java

License:Apache License

@Override
public void write(Batch batch) throws StageException {
    Map<HiveEndPoint, TransactionBatch> transactionBatches = new HashMap<>();
    Iterator<Record> it = batch.getRecords();

    while (it.hasNext()) {
        Record record = it.next();//from w  w w. j a  va 2s.c  om

        // Check that record has all required fields (partition columns).
        List<String> missingPartitions = getMissingRequiredFields(record, partitionsToFields);

        if (missingPartitions.size() == 0) {
            try {
                HiveEndPoint endPoint = getEndPointForRecord(record);

                TransactionBatch hiveBatch = transactionBatches.get(endPoint);
                if (null == hiveBatch || 0 == hiveBatch.remainingTransactions()) {
                    hiveBatch = getBatch(txnBatchSize, endPoint);
                    transactionBatches.put(endPoint, hiveBatch);
                }

                hiveBatch.beginNextTransaction();

                ByteArrayOutputStream bytes = new ByteArrayOutputStream(bufferLimit);
                DataGenerator generator = dataGeneratorFactory.getGenerator(bytes);

                // Transform record for field mapping overrides
                applyCustomMappings(record);

                // Remove Partition fields
                for (String fieldPath : partitionsToFields.values()) {
                    record.delete(fieldPath);
                }
                generator.write(record);
                generator.close();

                hiveBatch.write(bytes.toByteArray());
                hiveBatch.commit();
            } catch (InterruptedException | StreamingException | IOException e) {
                LOG.error("Error processing batch: {}", e.toString(), e);
                throw new StageException(Errors.HIVE_01, e.toString(), e);
            } catch (ExecutionException e) {
                LOG.error("Error processing batch: {}", e.getCause().toString(), e);
                throw new StageException(Errors.HIVE_01, e.getCause().toString(), e);
            } catch (OnRecordErrorException e) {
                handleError(record, e.getErrorCode(), e.getParams());
            }
        } else {
            if (missingPartitions.size() != 0) {
                handleError(record, Errors.HIVE_08, StringUtils.join(",", missingPartitions));
            }
        }
    }

    for (TransactionBatch transactionBatch : transactionBatches.values()) {
        try {
            transactionBatch.close();
        } catch (InterruptedException | StreamingException e) {
            LOG.error("Failed to close transaction batch: {}", e.toString(), e);
        }
    }
}

From source file:com.wandisco.s3hdfs.rewrite.redirect.MultiPartFileRedirect.java

License:Apache License

private void doIncrementalConcat(List<String> sources) throws IOException {
    int sourcesToConcat = sources.size();
    int increments = sourcesToConcat / 500;

    int i = 0;/*www . j  a  v  a  2  s  .c  o m*/
    do {
        int startIndex = (i * 500 == 0) ? 1 : i * 500; //1, 500, 1000, 1500...
        int endIndex = ((i + 1) * 500); //499, 999, 1499...
        if (endIndex >= sourcesToConcat)
            endIndex = sourcesToConcat;
        List<String> toConcat = sources.subList(startIndex, endIndex);
        System.out.println("CONCAT SRCS[" + i + "]: " + toConcat.toString());
        String conCatSrcs = StringUtils.join(",", toConcat);

        PostMethod httpPost = (PostMethod) getHttpMethod(request.getScheme(), request.getServerName(),
                request.getServerPort(), "CONCAT&sources=" + conCatSrcs, path.getUserName(),
                ADD_WEBHDFS(path.getHdfsRootUploadPath() + "1" + PART_FILE_NAME), POST);

        httpClient.executeMethod(httpPost);
        httpPost.releaseConnection();
        assert httpPost.getStatusCode() == 200;

        i++;
    } while (i <= increments);
}

From source file:com.wandisco.s3hdfs.rewrite.redirect.VersionRedirect.java

License:Apache License

/**
 * Checks the bucket to see if versioning is enabled or not.
 *
 * @param nnHostAddress//from  ww  w. j  av  a  2 s .  c  om
 * @param userName
 * @return
 */
public boolean check(String nnHostAddress, String userName) throws IOException {
    String[] nnHost = nnHostAddress.split(":");

    String uri = (path != null) ? path.getFullHdfsObjPath() : request.getPathInfo();
    List<String> bucketPath = Arrays.asList(uri.split("/")).subList(0, 4);
    String bucketMetaPath = StringUtils.join("/", bucketPath) + "/" + BUCKET_META_FILE_NAME;

    GetMethod httpGet = (GetMethod) getHttpMethod(request.getScheme(), nnHost[0], Integer.decode(nnHost[1]),
            "OPEN", userName, bucketMetaPath, GET);

    httpClient.executeMethod(httpGet);

    String versionConfXml = readInputStream(httpGet.getResponseBodyAsStream());
    httpGet.releaseConnection();

    if (versionConfXml.contains("<Status>Enabled</Status>")) {
        return true;
    } else if (versionConfXml.contains("<Status>Suspended</Status>")) {
        return false;
    } else {
        return false;
    }
}

From source file:eagle.jobrunning.ha.HAURLSelectorImpl.java

License:Apache License

@Override
public void reSelectUrl() throws IOException {
    if (reselectInProgress)
        return;/*from w w  w . j  av a  2 s . c  om*/
    synchronized (this) {
        if (reselectInProgress)
            return;
        reselectInProgress = true;
        try {
            LOG.info("Going to reselect url");
            for (int i = 0; i < urls.length; i++) {
                String urlToCheck = urls[i];
                LOG.info("Going to try url :" + urlToCheck);
                for (int time = 0; time < MAX_RETRY_TIME; time++) {
                    if (checkUrl(builder.build(urlToCheck, JobState.RUNNING.name()))) {
                        selectedUrl = urls[i];
                        LOG.info("Successfully switch to new url : " + selectedUrl);
                        return;
                    }
                    LOG.info("try url " + urlToCheck + "fail for " + (time + 1)
                            + " times, sleep 5 seconds before try again. ");
                    try {
                        Thread.sleep(5 * 1000);
                    } catch (InterruptedException ex) {
                        /* Do Nothing */}
                }
            }
            throw new IOException("No alive url found: " + StringUtils.join(";", Arrays.asList(this.urls)));
        } finally {
            reselectInProgress = false;
        }
    }
}

From source file:ezbake.amino.job.bitmap.EzAzkabanBitmapJobLoader.java

License:Apache License

/**
 * Azkaban's runner will call the no parameter run method.  Typical command line arguments are passed in the
 * constructor and are stored in the 'args' variable.
 *//*w  w w  .ja v  a 2  s  .  c o m*/
@SuppressWarnings("unused")
public void run() {
    final Configuration conf = new Configuration();
    final String jobDir = EzJobUtil.BITMAPS_DIR + projectId + "/" + startTimestamp + "/";
    Class<? extends BitmapJob> bitmapJobClass;
    int res;

    try {
        // Load up any EzBake specific configurations (Accumulo info, etc)
        EzJobUtil.loadEzConfigurations(conf);

        // Set the umask so that when we create the directories they will be readable by the group so they can be deleted later
        conf.set("fs.permissions.umask-mode", "002");

        // Set the base dir for the current job so we know where to make the working dir, etc
        conf.set(AminoConfiguration.BASE_DIR, jobDir);
        logger.info("Set job base dir to: " + jobDir);

        // Find the most recent run of all the different jobs
        final String baseDirs = StringUtils.join(",", EzJobUtil.findMostRecentRuns(conf));
        conf.set(AminoConfiguration.OUTPUT_DIR, baseDirs);
        logger.info("Set most recent analytic runs: " + baseDirs);

        // Figure out which job to load and run it
        bitmapJobClass = Class.forName(bitmapClass).asSubclass(BitmapJob.class);
        conf.set("mapreduce.job.name", String.format("%s_%s_%s", conf.get("application.name", "Batch_Bitmaps"),
                conf.get("service.name", "bitmapProcessing"), bitmapJobClass.getSimpleName()));
        res = ToolRunner.run(conf, bitmapJobClass.newInstance(), args);
    } catch (ClassNotFoundException e) {
        logger.error("Could not find the class: " + properties.getProperty("BitmapJob.class",
                "<property BitmapJob.class not found in properties"));
        try {
            FrameworkDriver.updateStatus(conf, FrameworkDriver.JobStatus.FAILED, new Path(jobDir));
        } catch (IOException ex) {
            // Swallow exception
        }
        res = -1;
    } catch (Exception e) {
        logger.error("Could not instantiate the class", e);
        try {
            FrameworkDriver.updateStatus(conf, FrameworkDriver.JobStatus.FAILED, new Path(jobDir));
        } catch (IOException ex) {
            // Swallow exception
        }
        res = -1;
    }

    System.exit(res);
}

From source file:ezbatch.amino.job.bitmap.EzAzkabanBitmapJobLoader.java

License:Apache License

/**
 * Azkaban's runner will call the no parameter run method.  Typical command line arguments are passed in the
 * constructor and are stored in the 'args' variable.
 *///  ww w.j a v  a2  s. c o  m
@SuppressWarnings("unused")
public void run() {
    final Configuration conf = new Configuration();
    final String jobDir = EzJobUtil.BITMAPS_DIR + projectId + "/" + startTimestamp + "/";
    Class<? extends BitmapJob> bitmapJobClass;
    int res;

    try {
        // Load up any Ezbake specific configurations (Accumulo info, etc)
        EzJobUtil.loadEzbakeConfigurations(conf);

        // Set the umask so that when we create the directories they will be readable by the group so they can be deleted later
        conf.set("fs.permissions.umask-mode", "002");

        // Set the base dir for the current job so we know where to make the working dir, etc
        conf.set(AminoConfiguration.BASE_DIR, jobDir);
        logger.info("Set job base dir to: " + jobDir);

        // Find the most recent run of all the different jobs
        final String baseDirs = StringUtils.join(",", EzJobUtil.findMostRecentRuns(conf));
        conf.set(AminoConfiguration.OUTPUT_DIR, baseDirs);
        logger.info("Set most recent analytic runs: " + baseDirs);

        // Figure out which job to load and run it
        bitmapJobClass = Class.forName(bitmapClass).asSubclass(BitmapJob.class);
        conf.set("mapreduce.job.name", String.format("%s_%s_%s", conf.get("application.name", "EzBatch"),
                conf.get("service.name", "bitmapProcessing"), bitmapJobClass.getSimpleName()));
        res = ToolRunner.run(conf, bitmapJobClass.newInstance(), args);
    } catch (ClassNotFoundException e) {
        logger.error("Could not find the class: " + properties.getProperty("BitmapJob.class",
                "<property BitmapJob.class not found in properties"));
        try {
            FrameworkDriver.updateStatus(conf, FrameworkDriver.JobStatus.FAILED, new Path(jobDir));
        } catch (IOException ex) {
            // Swallow exception
        }
        res = -1;
    } catch (Exception e) {
        logger.error("Could not instantiate the class", e);
        try {
            FrameworkDriver.updateStatus(conf, FrameworkDriver.JobStatus.FAILED, new Path(jobDir));
        } catch (IOException ex) {
            // Swallow exception
        }
        res = -1;
    }

    System.exit(res);
}

From source file:io.amient.yarn1.YarnContainerContext.java

License:Open Source License

private List<String> prepareCommands() {
    String command = "java " + jvmArgs + " -cp $CLASSPATH:./" + jarName + " " + mainClassName + " "
            + StringUtils.join(" ", args);
    command += " 1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stdout";
    command += " 2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stderr";
    log.info("$COMMAND = " + command);
    return Arrays.asList(command);
}