Example usage for org.apache.hadoop.fs Path Path

List of usage examples for org.apache.hadoop.fs Path Path

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.fs.*
  5. Path
  6. Path

Introduction

In this page you can find the example usage for org.apache.hadoop.fs Path Path.

Prototype

public Path(URI aUri)

Source Link

Document

Construct a path from a URI

Usage

From source file:a.b.c.MultiFileWordCount.java

License:Apache License

public int run(String[] args) throws Exception {

    if (args.length < 2) {
        printUsage();//from   www . j av a  2  s .  com
        return 2;
    }

    Job job = new Job(getConf());
    job.setJobName("MultiFileWordCount");
    job.setJarByClass(MultiFileWordCount.class);

    //set the InputFormat of the job to our InputFormat
    job.setInputFormatClass(MyInputFormat.class);

    // the keys are words (strings)
    job.setOutputKeyClass(Text.class);
    // the values are counts (ints)
    job.setOutputValueClass(IntWritable.class);

    //use the defined mapper
    job.setMapperClass(MapClass.class);
    //use the WordCount Reducer
    job.setCombinerClass(IntSumReducer.class);
    job.setReducerClass(IntSumReducer.class);

    FileInputFormat.addInputPaths(job, args[0]);
    FileOutputFormat.setOutputPath(job, new Path(args[1]));

    return job.waitForCompletion(true) ? 0 : 1;
}

From source file:a.TestConcatExample.java

License:Apache License

@Test
public void concatIsPermissive() throws IOException, URISyntaxException {
    MiniDFSCluster cluster = null;/*from  ww  w.j a  va2s .co m*/
    final Configuration conf = WebHdfsTestUtil.createConf();
    conf.set("dfs.namenode.fs-limits.min-block-size", "1000"); // Allow tiny blocks for the test
    try {
        cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build();
        cluster.waitActive();
        final FileSystem webHdfs = WebHdfsTestUtil.getWebHdfsFileSystem(conf, WebHdfsFileSystem.SCHEME);
        final FileSystem dfs = cluster.getFileSystem();

        final FileSystem fs = dfs; // WebHDFS has a bug in getLocatedBlocks

        Path root = new Path("/dir");
        fs.mkdirs(root);

        short origRep = 3;
        short secondRep = (short) (origRep - 1);
        Path f1 = new Path("/dir/f1");
        long size1 = writeFile(fs, f1, /* blocksize */ 4096, origRep, 5);
        long f1NumBlocks = fs.getFileBlockLocations(f1, 0, size1).length;
        assertEquals(5, f1NumBlocks);

        Path f2 = new Path("/dir/f2");
        long size2 = writeFile(fs, f2, /* blocksize (must divide 512 for checksum) */ 4096 - 512, secondRep, 4);
        long f2NumBlocks = fs.getFileBlockLocations(f2, 0, size2).length;
        assertEquals(5, f2NumBlocks);

        fs.concat(f1, new Path[] { f2 });
        FileStatus[] fileStatuses = fs.listStatus(root);

        // Only one file should remain
        assertEquals(1, fileStatuses.length);
        FileStatus fileStatus = fileStatuses[0];

        // And it should be named after the first file
        assertEquals("f1", fileStatus.getPath().getName());

        // The entire file takes the replication of the first argument
        assertEquals(origRep, fileStatus.getReplication());

        // As expected, the new concated file is the length of both the previous files
        assertEquals(size1 + size2, fileStatus.getLen());

        // And we should have the same number of blocks
        assertEquals(f1NumBlocks + f2NumBlocks,
                fs.getFileBlockLocations(fileStatus.getPath(), 0, size1 + size2).length);
    } finally {
        if (cluster != null) {
            cluster.shutdown();
        }

    }
}

From source file:abtGlobals.globalVARArea.java

public globalVARArea(String metaDataConfPath) { //Constructor Carga Datos Iniciales
    Properties srvConf = new Properties();
    cfg_glb_metaDataConfPath = metaDataConfPath;
    String hbParamConf;//from   w  w w.j av a2 s .co m

    isGetSrvConf = false;
    try {
        //Extrae Fecha de Hoy

        Date today;
        String output;
        SimpleDateFormat formatter;

        formatter = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
        today = new Date();
        output = formatter.format(today);

        cfg_srv_FecInicio = output;

        srvConf.load(new FileInputStream(metaDataConfPath));
        //
        // Recupera Parametros Base de Datos SrvViewer en HBAse
        cfg_glb_DBType = srvConf.getProperty("DBType");
        hbParamConf = srvConf.getProperty("HBConfVar");

        if (cfg_glb_DBType.equals("HBASE")) {
            cfg_glb_hbConfFile = srvConf.getProperty(hbParamConf).split(",");
            cfg_glb_hbTableName = srvConf.getProperty("HBTable");
            for (int i = 0; i < cfg_glb_hbConfFile.length; i++) {
                cfg_glb_hcfg.addResource(new Path(cfg_glb_hbConfFile[i]));
            }
        }

        //Recupera Parametros de Conexion a BD en ORA y SQL
        if (cfg_glb_DBType.equals("ORA")) {
            cfg_db_Host = srvConf.getProperty("ORA_DBHost");
            cfg_db_Name = srvConf.getProperty("ORA_DBName");
            cfg_db_User = srvConf.getProperty("ORA_DBUser");
            cfg_db_Pass = srvConf.getProperty("ORA_DBPass");
            cfg_db_Port = srvConf.getProperty("ORA_DBPort");
            cfg_db_Table = srvConf.getProperty("ORA_DBTable");

        }

        if (cfg_glb_DBType.equals("SQL")) {
            cfg_db_Host = srvConf.getProperty("SQL_DBHost");
            cfg_db_Name = srvConf.getProperty("SQL_DBName");
            cfg_db_User = srvConf.getProperty("SQL_DBUser");
            cfg_db_Pass = srvConf.getProperty("SQL_DBPass");
            cfg_db_Port = srvConf.getProperty("SQL_DBPort");
            cfg_db_Table = srvConf.getProperty("SQL_DBTable");

        }

        //Recupera Parametros del Servicio
        cfg_srv_ID = srvConf.getProperty("SRV_ID");
        cfg_srv_Name = srvConf.getProperty("SRV_DESC");
        cfg_srv_Online = srvConf.getProperty("SRV_ONLINE");
        cfg_srv_Txp_Control = srvConf.getProperty("SRV_TXP_CONTROL");
        cfg_srv_Txp_Keep = srvConf.getProperty("SRV_TXP_KEEP");
        cfg_srv_Txp_Proc = srvConf.getProperty("SRV_TXP_PROC");
        cfg_srv_Log_Active = srvConf.getProperty("SRV_LOG_ACTIVE");
        cfg_srv_Log_Dir = srvConf.getProperty("SRV_LOG_DIR");
        cfg_srv_Max_thProcess = srvConf.getProperty("SRV_MAX_THPROCESS");
        cfg_srv_type_proc = srvConf.getProperty("SRV_TYPE_PROC").split(",");
        cfg_srv_host_auth = srvConf.getProperty("SRV_HOST_AUTH").split(",");
        cfg_srv_Log_Name = srvConf.getProperty("SRV_LOG_NAME");

        try {
            cfg_glb_hostName = InetAddress.getLocalHost().getHostName();
        } catch (UnknownHostException ex) {
            cfg_glb_hostName = "NOT_FOUND";
            isGetSrvConf = false;
            //Logger.getLogger(ttModuleControl.class.getName()).log(Level.SEVERE, null, ex);
            logger.error(ex.getMessage());
        }

        //Activa logs
        //
        PropertyConfigurator.configure("log4j.properties");

        isGetSrvConf = true;

    } catch (FileNotFoundException e) {
        isGetSrvConf = false;
        gErrorMessages = e.getMessage();
        gErrorNumber = 99;
    } catch (IOException e) {
        isGetSrvConf = false;
        gErrorMessages = e.getMessage();
        gErrorNumber = 99;
    }
}

From source file:accesslog2.Accesslog2.java

public static void main(String[] args) throws Exception {
    Configuration conf = new Configuration();
    Path inputPath = new Path(args[0]);
    Path outputPath = new Path(args[1]);
    Job job = Job.getInstance(conf, "Accesslog2");
    job.setJarByClass(Accesslog2.class);
    job.setMapperClass(TokenizerMapper.class);
    job.setCombinerClass(IntSumReducer.class);
    job.setReducerClass(IntSumReducer.class);
    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(IntWritable.class);
    FileInputFormat.addInputPath(job, inputPath);
    FileOutputFormat.setOutputPath(job, outputPath);
    job.setInputFormatClass(TextInputFormat.class);
    job.setOutputFormatClass(TextOutputFormat.class);
    System.exit(job.waitForCompletion(true) ? 0 : 1);
}

From source file:accesslog3.Accesslog3.java

public static void main(String[] args) throws Exception {
    Configuration conf = new Configuration();
    Path inputPath = new Path(args[0]);
    Path outputPath = new Path(args[1]);
    Job job = Job.getInstance(conf, "Accesslog3");
    job.setJarByClass(Accesslog3.class);
    job.setMapperClass(TokenizerMapper.class);
    job.setCombinerClass(IntSumReducer.class);
    job.setReducerClass(IntSumReducer.class);
    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(IntWritable.class);
    FileInputFormat.addInputPath(job, inputPath);
    FileOutputFormat.setOutputPath(job, outputPath);
    job.setInputFormatClass(TextInputFormat.class);
    job.setOutputFormatClass(TextOutputFormat.class);
    System.exit(job.waitForCompletion(true) ? 0 : 1);
}

From source file:accesslog4.Accesslog4.java

public static void main(String[] args) throws Exception {
    Configuration conf = new Configuration();
    Path inputPath = new Path(args[0]);
    Path outputPath = new Path(args[1]);
    Job job = Job.getInstance(conf, "Accesslog4");
    job.setJarByClass(Accesslog4.class);
    job.setMapperClass(TokenizerMapper.class);
    job.setCombinerClass(IntSumReducer.class);
    job.setReducerClass(IntSumReducer.class);
    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(IntWritable.class);
    FileInputFormat.addInputPath(job, inputPath);
    FileOutputFormat.setOutputPath(job, outputPath);
    job.setInputFormatClass(TextInputFormat.class);
    job.setOutputFormatClass(TextOutputFormat.class);
    System.exit(job.waitForCompletion(true) ? 0 : 1);
}

From source file:accismus.benchmark.Verifier.java

License:Apache License

@Override
public int run(String[] args) throws Exception {

    if (args.length != 2) {
        System.err.println("Usage : " + this.getClass().getSimpleName() + " <props file> <output dir>");
        return 1;
    }//from w  w  w. j  a  v a  2s  .  com

    Job job = new Job(getConf(), this.getClass().getSimpleName() + "_" + System.currentTimeMillis());
    job.setJarByClass(this.getClass());

    AccismusProperties accisumusProps = new AccismusProperties(new File(args[0]));

    AccismusInputFormat.configure(job, accisumusProps);
    AccismusInputFormat.fetchFamilies(job, KEY, DUP);

    job.setInputFormatClass(AccismusInputFormat.class);

    job.setMapOutputKeyClass(Text.class);
    job.setMapOutputValueClass(Text.class);

    job.setMapperClass(VMapper.class);

    job.setReducerClass(VReducer.class);

    job.setOutputFormatClass(TextOutputFormat.class);

    TextOutputFormat.setOutputPath(job, new Path(args[1]));

    job.waitForCompletion(true);

    return 0;
}

From source file:acromusashi.stream.bolt.hdfs.HdfsOutputSwitcher.java

License:Open Source License

/**
 * ?????????????/*from  ww  w  .  j  a v  a  2s.c o  m*/
 */
private void closeRenameTmp2BaseFile() {
    try {
        this.currentWriter.close();
    } catch (IOException ex) {
        String logFormat = "Failed to HDFS file close. Continue file switch. : TargetUri={0}";
        String logMessage = MessageFormat.format(logFormat, this.currentOutputUri + this.currentSuffix);
        logger.warn(logMessage, ex);
    }

    boolean isFileExists = true;

    try {
        isFileExists = this.fileSystem.exists(new Path(this.currentOutputUri));
    } catch (IOException ioex) {
        String logFormat = "Failed to search target file exists. Skip file rename. : TargetUri={0}";
        String logMessage = MessageFormat.format(logFormat, this.currentOutputUri);
        logger.warn(logMessage, ioex);
        return;
    }

    if (isFileExists) {
        String logFormat = "File exists renamed target. Skip file rename. : BeforeUri={0} , AfterUri={1}";
        String logMessage = MessageFormat.format(logFormat, this.currentOutputUri + this.currentSuffix,
                this.currentOutputUri);
        logger.warn(logMessage);
    } else {
        try {
            this.fileSystem.rename(new Path(this.currentOutputUri + this.currentSuffix),
                    new Path(this.currentOutputUri));
        } catch (IOException ex) {
            String logFormat = "Failed to HDFS file rename. Skip rename file. : BeforeUri={0} , AfterUri={1}";
            String logMessage = MessageFormat.format(logFormat, this.currentOutputUri + this.currentSuffix,
                    this.currentOutputUri);
            logger.warn(logMessage, ex);
        }
    }

}

From source file:acromusashi.stream.bolt.hdfs.HdfsPreProcessor.java

License:Open Source License

/**
 * HDFSSink????<br>/*from w ww .  ja va  2  s  .c  o  m*/
 * ?????????????????
 * 
 * @param hdfs 
 * @param baseUrl URL
 * @param baseName ??
 * @param tmpSuffix ??
 */
public static void execute(FileSystem hdfs, String baseUrl, String baseName, String tmpSuffix) {
    String baseRealUrl = baseUrl;

    if (baseRealUrl.endsWith("/") == false) {
        baseRealUrl = baseRealUrl + "/";
    }

    String targetPattern = baseRealUrl + baseName + "[0-9]*" + tmpSuffix + "*";
    Path targetPathPattern = new Path(targetPattern);

    FileStatus[] targetTmpFiles = null;

    try {
        targetTmpFiles = hdfs.globStatus(targetPathPattern);
    } catch (IOException ioex) {
        logger.warn("Failed to search preprocess target files. Skip preprocess.", ioex);
        return;
    }

    if (targetTmpFiles.length == 0) {
        String logFormat = "Preprocess target files not exist. Path={0}";
        String logMessage = MessageFormat.format(logFormat, targetPattern);
        logger.info(logMessage);
        return;
    }

    if (logger.isInfoEnabled() == true) {
        printTargetPathList(targetTmpFiles);
    }

    for (FileStatus targetTmpFile : targetTmpFiles) {
        renameTmpFile(hdfs, targetTmpFile.getPath().toString(), tmpSuffix);
    }

}

From source file:acromusashi.stream.bolt.hdfs.HdfsPreProcessor.java

License:Open Source License

/**
 * ????<br>//  ww  w .j ava  2  s  . c  o m
 * ???????????
 * 
 * @param hdfs 
 * @param targetTmpPath ??
 * @param tmpSuffix ??
 */
private static void renameTmpFile(FileSystem hdfs, String targetTmpPath, String tmpSuffix) {
    String basePath = extractBasePath(targetTmpPath, tmpSuffix);

    boolean isFileExists = true;

    try {
        isFileExists = hdfs.exists(new Path(basePath));
    } catch (IOException ioex) {
        String logFormat = "Failed to search target file exists. Skip file rename. : TargetUri={0}";
        String logMessage = MessageFormat.format(logFormat, basePath);
        logger.warn(logMessage, ioex);
        return;
    }

    if (isFileExists) {
        String logFormat = "File exists renamed target. Skip file rename. : BeforeUri={0} , AfterUri={1}";
        String logMessage = MessageFormat.format(logFormat, targetTmpPath, basePath);
        logger.warn(logMessage);
    } else {
        try {
            hdfs.rename(new Path(targetTmpPath), new Path(basePath));
        } catch (IOException ioex) {
            String logFormat = "Failed to HDFS file rename. Skip rename file and continue preprocess. : BeforeUri={0} , AfterUri={1}";
            String logMessage = MessageFormat.format(logFormat, targetTmpPath, basePath);
            logger.warn(logMessage, ioex);
        }
    }
}