Example usage for org.apache.hadoop.fs Path toString

Introduction

In this page you can find the example usage for org.apache.hadoop.fs Path toString.

Prototype

@Override
    public String toString() 

Source Link

Usage

From source file:com.hive_unit.EmbeddedHiveExampleTest.java

License:Apache License

public void testEmbeddedHive() throws Exception {
    Path p = new Path(this.ROOT_DIR, "afile");

    FSDataOutputStream o = this.getFileSystem().create(p);
    BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(o));
    bw.write("1\n");
    bw.write("2\n");
    bw.close();//from  w  ww.j a v a  2 s . c o  m

    JobConf c = createJobConf();
    Assert.assertEquals(0, doHiveCommand("create table bla (id int)", c));
    Assert.assertEquals(0, doHiveCommand("load data local inpath '" + p.toString() + "' into table bla", c));
    Assert.assertEquals(0, doHiveCommand("create table blb (id int)", c));
    Assert.assertEquals(1, doHiveCommand("create table bla (id int)", c));
    Assert.assertEquals(0, doHiveCommand("select * from bla", c));
    Assert.assertEquals(0, doHiveCommand("select count(1) from bla", c));
    Assert.assertEquals(0, doHiveCommand("select id from bla", c));

}

---

From source file:com.hive_unit.EmbeddedHiveTest.java

License:Apache License

public void testUseEmbedded() throws IOException {
    EmbeddedHive eh = new EmbeddedHive(TestingUtil.getDefaultProperties());
    Path p = new Path(this.ROOT_DIR, "bfile");

    FSDataOutputStream o = this.getFileSystem().create(p);
    BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(o));
    bw.write("1\n");
    bw.write("2\n");
    bw.close();//from   www .j  ava  2  s  . c om

    assertEquals(SUCCESS, eh.doHiveCommand("create table blax (id int)").getResponseStatus());
    assertEquals(SUCCESS, eh.doHiveCommand("load data local inpath '" + p.toString() + "' into table blax")
            .getResponseStatus());
    assertEquals(SUCCESS, eh.doHiveCommand("create table blbx (id int)").getResponseStatus());
    assertEquals(FAILURE, eh.doHiveCommand("create table blax (id int)").getResponseStatus());
    assertEquals(SUCCESS, eh.doHiveCommand("select count(1) from blax").getResponseStatus());
}

---

From source file:com.hive_unit.ServiceExampleTest.java

License:Apache License

public void testExecute() throws Exception {
    Path p = new Path(this.ROOT_DIR, "afile");

    FSDataOutputStream o = this.getFileSystem().create(p);
    BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(o));
    bw.write("1\n");
    bw.write("2\n");
    bw.close();// ww w. ja v a 2s . c o  m

    client.execute("create table  atest  (num int)");
    client.execute("load data local inpath '" + p.toString() + "' into table atest");
    client.execute("select count(1) as cnt from atest");
    String row = client.fetchOne();
    assertEquals(row, "2");
    client.execute("drop table atest");
}

---

From source file:com.hive_unit.ServiceHiveTest.java

License:Apache License

public void testExecute() throws Exception {
    Path p = new Path(this.ROOT_DIR, "afile");

    FSDataOutputStream o = this.getFileSystem().create(p);
    BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(o));
    bw.write("1\n");
    bw.write("2\n");
    bw.close();// w w w  .  j  a v a  2  s . c  o  m
    ServiceHive sh = new ServiceHive();

    sh.client.execute("create table  atest  (num int)");
    sh.client.execute("load data local inpath '" + p.toString() + "' into table atest");
    sh.client.execute("select count(1) as cnt from atest");
    String row = sh.client.fetchOne();
    assertEquals("2", row);
    sh.client.execute("drop table atest");
}

---

From source file:com.hortonworks.minicluster.InJvmContainerExecutor.java

License:Apache License

/**
 * Will launch containers within the same JVM as this Container Executor. It
 * will do so by: - extracting Container's class name and program arguments
 * from the launch script (e.g., launch_container.sh) - Creating an isolated
 * ClassLoader for each container - Calling doLaunchContainer(..) method to
 * launch Container/*w  w  w . j av  a  2s . c  om*/
 */
private int doLaunch(Container container, Path containerWorkDir) {
    Map<String, String> environment = container.getLaunchContext().getEnvironment();
    EnvironmentUtils.putAll(environment);

    Set<URL> additionalClassPathUrls = this.filterAndBuildUserClasspath(container);

    ExecJavaCliParser javaCliParser = this.createExecCommandParser(containerWorkDir.toString());

    UserGroupInformation.setLoginUser(null);
    try {
        // create Isolated Class Loader for each container and set it as context
        // class loader
        URLClassLoader containerCl = new URLClassLoader(
                additionalClassPathUrls.toArray(additionalClassPathUrls.toArray(new URL[] {})), null);
        Thread.currentThread().setContextClassLoader(containerCl);
        String containerLauncher = javaCliParser.getMain();
        Class<?> containerClass = Class.forName(containerLauncher, true, containerCl);
        Method mainMethod = containerClass.getMethod("main", new Class[] { String[].class });
        mainMethod.setAccessible(true);
        String[] arguments = javaCliParser.getMainArguments();

        this.doLaunchContainer(containerClass, mainMethod, arguments);
    } catch (Exception e) {
        logger.error("Failed to launch container " + container, e);
        container.handle(new ContainerDiagnosticsUpdateEvent(container.getContainerId(), e.getMessage()));
        return -1;
    } finally {
        logger.info("Removing symlinks");
        this.cleanUp();
    }
    return 0;
}

---

From source file:com.hortonworks.registries.common.util.HdfsFileStorage.java

License:Apache License

@Override
public String uploadFile(InputStream inputStream, String name) throws IOException {
    Path jarPath = new Path(directory, name);

    try (FSDataOutputStream outputStream = hdfsFileSystem.create(jarPath, false)) {
        ByteStreams.copy(inputStream, outputStream);
    }//ww  w. j  a va2s .c  o m

    return jarPath.toString();
}

---

From source file:com.hortonworks.streamline.common.util.HdfsFileStorage.java

License:Apache License

@Override
public String upload(InputStream inputStream, String name) throws IOException {
    Path jarPath = new Path(directory, name);

    try (FSDataOutputStream outputStream = hdfsFileSystem.create(jarPath, false)) {
        ByteStreams.copy(inputStream, outputStream);
    }//from w  w  w  .ja va 2  s.c o  m

    return jarPath.toString();
}

---

From source file:com.ibm.bi.dml.conf.DMLConfig.java

License:Open Source License

/**
 * //  w w w .  j ava2s  .co  m
 * @throws IOException
 */
@SuppressWarnings("deprecation")
public void makeQualifiedScratchSpacePath() throws IOException {
    NodeList list2 = xml_root.getElementsByTagName(SCRATCH_SPACE);
    if (list2 != null && list2.getLength() > 0) {
        Element elem = (Element) list2.item(0);

        FileSystem fs = FileSystem.get(ConfigurationManager.getCachedJobConf());
        String fname = elem.getFirstChild().getNodeValue();
        Path path = new Path(fname).makeQualified(fs);

        elem.getFirstChild().setNodeValue(path.toString());
    }
}

---

From source file:com.ibm.bi.dml.lops.runtime.RunMRJobs.java

License:Open Source License

/**
 * Submits an MR job instruction, without modifying any state of that instruction.
 * /*from   w  w  w .j a v a 2  s  .  com*/
 * @param inst
 * @param ec
 * @return
 * @throws DMLRuntimeException
 */
public static JobReturn submitJob(MRJobInstruction inst) throws DMLRuntimeException {
    JobReturn ret = new JobReturn();
    MatrixObject[] inputMatrices = inst.getInputMatrices();
    MatrixObject[] outputMatrices = inst.getOutputMatrices();
    boolean execCP = false;

    // Spawn MapReduce Jobs
    try {
        // replace all placeholders in all instructions with appropriate values
        String rdInst = inst.getIv_randInstructions();
        String rrInst = inst.getIv_recordReaderInstructions();
        String mapInst = inst.getIv_instructionsInMapper();
        String shuffleInst = inst.getIv_shuffleInstructions();
        String aggInst = inst.getIv_aggInstructions();
        String otherInst = inst.getIv_otherInstructions();
        boolean jvmReuse = ConfigurationManager.getConfig().getBooleanValue(DMLConfig.JVM_REUSE);

        switch (inst.getJobType()) {

        case GMR:
        case GMRCELL:
            ret = GMR.runJob(inst, inst.getInputs(), inst.getInputInfos(), inst.getRlens(), inst.getClens(),
                    inst.getBrlens(), inst.getBclens(), inst.getPartitioned(), inst.getPformats(),
                    inst.getPsizes(), rrInst, mapInst, aggInst, otherInst, inst.getIv_numReducers(),
                    inst.getIv_replication(), jvmReuse, inst.getIv_resultIndices(),
                    inst.getDimsUnknownFilePrefix(), inst.getOutputs(), inst.getOutputInfos());
            break;

        case DATAGEN:
            if (OptimizerUtils.ALLOW_DYN_RECOMPILATION && OptimizerUtils.ALLOW_RAND_JOB_RECOMPILE
                    && DMLScript.rtplatform != RUNTIME_PLATFORM.HADOOP
                    && Recompiler.checkCPDataGen(inst, rdInst)) {
                ret = executeInMemoryDataGenOperations(inst, rdInst, outputMatrices);
                Statistics.decrementNoOfExecutedMRJobs();
                execCP = true;
            } else {
                ret = DataGenMR.runJob(inst, rdInst.split(Lop.INSTRUCTION_DELIMITOR), mapInst, aggInst,
                        otherInst, inst.getIv_numReducers(), inst.getIv_replication(),
                        inst.getIv_resultIndices(), inst.getDimsUnknownFilePrefix(), inst.getOutputs(),
                        inst.getOutputInfos());
            }
            break;

        case CM_COV:
            ret = CMCOVMR.runJob(inst, inst.getInputs(), inst.getInputInfos(), inst.getRlens(), inst.getClens(),
                    inst.getBrlens(), inst.getBclens(), mapInst, shuffleInst, inst.getIv_numReducers(),
                    inst.getIv_replication(), inst.getIv_resultIndices(), inst.getOutputs(),
                    inst.getOutputInfos());
            break;

        case GROUPED_AGG:
            ret = GroupedAggMR.runJob(inst, inst.getInputs(), inst.getInputInfos(), inst.getRlens(),
                    inst.getClens(), inst.getBrlens(), inst.getBclens(), shuffleInst, otherInst,
                    inst.getIv_numReducers(), inst.getIv_replication(), inst.getIv_resultIndices(),
                    inst.getDimsUnknownFilePrefix(), inst.getOutputs(), inst.getOutputInfos());
            break;

        case REBLOCK:
        case CSV_REBLOCK:
            if (OptimizerUtils.ALLOW_DYN_RECOMPILATION && DMLScript.rtplatform != RUNTIME_PLATFORM.HADOOP
                    && Recompiler.checkCPReblock(inst, inputMatrices)) {
                ret = executeInMemoryReblockOperations(inst, shuffleInst, inputMatrices, outputMatrices);
                Statistics.decrementNoOfExecutedMRJobs();
                execCP = true;
            } else {
                // export dirty matrices to HDFS (initially deferred)
                for (MatrixObject m : inputMatrices) {
                    if (m.isDirty())
                        m.exportData();
                }
                checkEmptyInputs(inst, inputMatrices);

                if (inst.getJobType() == JobType.REBLOCK) {
                    ret = ReblockMR.runJob(inst, inst.getInputs(), inst.getInputInfos(), inst.getRlens(),
                            inst.getClens(), inst.getBrlens(), inst.getBclens(), getNNZ(inputMatrices), mapInst,
                            shuffleInst, otherInst, inst.getIv_numReducers(), inst.getIv_replication(),
                            jvmReuse, inst.getIv_resultIndices(), inst.getOutputs(), inst.getOutputInfos());
                } else if (inst.getJobType() == JobType.CSV_REBLOCK) {
                    ret = CSVReblockMR.runJob(inst, inst.getInputs(), inst.getInputInfos(), inst.getRlens(),
                            inst.getClens(), inst.getBrlens(), inst.getBclens(), shuffleInst, otherInst,
                            inst.getIv_numReducers(), inst.getIv_replication(), inst.getIv_resultIndices(),
                            inst.getOutputs(), inst.getOutputInfos());
                }
            }
            break;

        case CSV_WRITE:
            ret = WriteCSVMR.runJob(inst, inst.getInputs(), inst.getInputInfos(), inst.getRlens(),
                    inst.getClens(), inst.getBclens(), inst.getBclens(), shuffleInst, inst.getIv_numReducers(),
                    inst.getIv_replication(), inst.getIv_resultIndices(), inst.getOutputs());
            break;

        case MMCJ:
            ret = MMCJMR.runJob(inst, inst.getInputs(), inst.getInputInfos(), inst.getRlens(), inst.getClens(),
                    inst.getBrlens(), inst.getBclens(), mapInst, aggInst, shuffleInst, inst.getIv_numReducers(),
                    inst.getIv_replication(), inst.getOutputs()[0], inst.getOutputInfos()[0]);
            break;

        case MMRJ:
            ret = MMRJMR.runJob(inst, inst.getInputs(), inst.getInputInfos(), inst.getRlens(), inst.getClens(),
                    inst.getBrlens(), inst.getBclens(), mapInst, aggInst, shuffleInst, otherInst,
                    inst.getIv_numReducers(), inst.getIv_replication(), inst.getIv_resultIndices(),
                    inst.getOutputs(), inst.getOutputInfos());
            break;

        case SORT:
            boolean weightsflag = true;
            if (!mapInst.equalsIgnoreCase(""))
                weightsflag = false;
            ret = SortMR.runJob(inst, inst.getInputs()[0], inst.getInputInfos()[0], inst.getRlens()[0],
                    inst.getClens()[0], inst.getBrlens()[0], inst.getBclens()[0], mapInst, shuffleInst,
                    inst.getIv_numReducers(), inst.getIv_replication(), inst.getOutputs()[0],
                    inst.getOutputInfos()[0], weightsflag);
            break;

        case COMBINE:
            ret = CombineMR.runJob(inst, inst.getInputs(), inst.getInputInfos(), inst.getRlens(),
                    inst.getClens(), inst.getBrlens(), inst.getBclens(), shuffleInst, inst.getIv_numReducers(),
                    inst.getIv_replication(), inst.getIv_resultIndices(), inst.getOutputs(),
                    inst.getOutputInfos());
            break;

        case DATA_PARTITION:
            ret = DataPartitionMR.runJob(inst, inputMatrices, shuffleInst, inst.getIv_resultIndices(),
                    outputMatrices, inst.getIv_numReducers(), inst.getIv_replication());
            break;

        case TRANSFORM:

            if (OptimizerUtils.ALLOW_DYN_RECOMPILATION && OptimizerUtils.ALLOW_TRANSFORM_RECOMPILE
                    && DMLScript.rtplatform != RUNTIME_PLATFORM.HADOOP
                    && Recompiler.checkCPTransform(inst, inputMatrices)) {
                // transform the data and generate output in CSV format
                ret = executeInMemoryTransform(inst, inputMatrices, outputMatrices);
                Statistics.decrementNoOfExecutedMRJobs();
                execCP = true;
            } else {
                ret = DataTransform.mrDataTransform(inst, inputMatrices, shuffleInst, otherInst,
                        inst.getIv_resultIndices(), outputMatrices, inst.getIv_numReducers(),
                        inst.getIv_replication());
            }
            break;

        default:
            throw new DMLRuntimeException("Invalid jobtype: " + inst.getJobType());
        }

    } // end of try block
    catch (Exception e) {
        throw new DMLRuntimeException(e);
    }

    if (ret.checkReturnStatus()) {
        /*
         * Check if any output is empty. If yes, create a dummy file. Needs
         * to be done only in case of (1) CellOutputInfo and if not CP, or 
         * (2) BinaryBlockOutputInfo if not CP and output empty blocks disabled.
         */
        try {
            if (!execCP) {
                for (int i = 0; i < outputMatrices.length; i++) {
                    //get output meta data
                    MatrixFormatMetaData meta = (MatrixFormatMetaData) outputMatrices[i].getMetaData();
                    MatrixCharacteristics mc = meta.getMatrixCharacteristics();
                    OutputInfo outinfo = meta.getOutputInfo();
                    String fname = outputMatrices[i].getFileName();

                    if (MapReduceTool.isHDFSFileEmpty(fname)) {
                        //prepare output file
                        Path filepath = new Path(fname, "0-m-00000");
                        MatrixWriter writer = MatrixWriterFactory.createMatrixWriter(outinfo);
                        writer.writeEmptyMatrixToHDFS(filepath.toString(), mc.getRows(), mc.getCols(),
                                mc.getRowsPerBlock(), mc.getColsPerBlock());
                    }

                    outputMatrices[i].setFileExists(true);

                    if (inst.getJobType() != JobType.CSV_WRITE && inst.getJobType() != JobType.TRANSFORM) {
                        // write out metadata file
                        // Currently, valueType information in not stored in MR instruction, 
                        // since only DOUBLE matrices are supported ==> hard coded the value type information for now
                        MapReduceTool.writeMetaDataFile(fname + ".mtd", ValueType.DOUBLE,
                                ((MatrixDimensionsMetaData) ret.getMetaData(i)).getMatrixCharacteristics(),
                                outinfo);
                    }
                }
            }
            return ret;
        } catch (IOException e) {
            throw new DMLRuntimeException(e);
        }
    }

    // should not come here!
    throw new DMLRuntimeException("Unexpected Job Type: " + inst.getJobType());
}

---

From source file:com.ibm.bi.dml.parser.DataExpression.java

License:Open Source License

/**
 * /*from ww  w  . j ava 2s .  c  o m*/
 * @param filename
 * @return
 * @throws LanguageException
 */
public JSONObject readMetadataFile(String filename, boolean conditional) throws LanguageException {
    JSONObject retVal = null;
    boolean exists = false;
    FileSystem fs = null;

    try {
        fs = FileSystem.get(ConfigurationManager.getCachedJobConf());
    } catch (Exception e) {
        raiseValidateError("could not read the configuration file: " + e.getMessage(), false);
    }

    Path pt = new Path(filename);
    try {
        if (fs.exists(pt)) {
            exists = true;
        }
    } catch (Exception e) {
        exists = false;
    }

    boolean isDirBoolean = false;
    try {
        if (exists && fs.getFileStatus(pt).isDirectory())
            isDirBoolean = true;
        else
            isDirBoolean = false;
    } catch (Exception e) {
        raiseValidateError(
                "error validing whether path " + pt.toString() + " is directory or not: " + e.getMessage(),
                conditional);
    }

    // CASE: filename is a directory -- process as a directory
    if (exists && isDirBoolean) {

        // read directory contents
        retVal = new JSONObject();

        FileStatus[] stats = null;

        try {
            stats = fs.listStatus(pt);
        } catch (Exception e) {
            raiseValidateError("for MTD file in directory, error reading directory with MTD file "
                    + pt.toString() + ": " + e.getMessage(), conditional);
        }

        for (FileStatus stat : stats) {
            Path childPath = stat.getPath(); // gives directory name
            if (childPath.getName().startsWith("part")) {

                BufferedReader br = null;
                try {
                    br = new BufferedReader(new InputStreamReader(fs.open(childPath)));
                } catch (Exception e) {
                    raiseValidateError("for MTD file in directory, error reading part of MTD file with path "
                            + childPath.toString() + ": " + e.getMessage(), conditional);
                }

                JSONObject childObj = null;
                try {
                    childObj = JSONHelper.parse(br);
                } catch (Exception e) {
                    raiseValidateError("for MTD file in directory, error parsing part of MTD file with path "
                            + childPath.toString() + ": " + e.getMessage(), conditional);
                }

                for (Object obj : childObj.entrySet()) {
                    @SuppressWarnings("unchecked")
                    Entry<Object, Object> e = (Entry<Object, Object>) obj;
                    Object key = e.getKey();
                    Object val = e.getValue();
                    retVal.put(key, val);
                }
            }
        } // end for 
    }

    // CASE: filename points to a file
    else if (exists) {

        BufferedReader br = null;

        // try reading MTD file
        try {
            br = new BufferedReader(new InputStreamReader(fs.open(pt)));
        } catch (Exception e) {
            raiseValidateError("error reading MTD file with path " + pt.toString() + ": " + e.getMessage(),
                    conditional);
        }

        // try parsing MTD file
        try {
            retVal = JSONHelper.parse(br);
        } catch (Exception e) {
            raiseValidateError("error parsing MTD file with path " + pt.toString() + ": " + e.getMessage(),
                    conditional);
        }
    }

    return retVal;
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• 37
• 38
• 39
• 40
• 41
• 42
• 43
• 44
• 45
• 46
• 47
• 48
• 49
• 50
• »