Example usage for org.apache.hadoop.fs Path Path

Introduction

In this page you can find the example usage for org.apache.hadoop.fs Path Path.

Prototype

public Path(URI aUri) 

Source Link

Document

Construct a path from a URI

Usage

From source file:Analysis.A2_Top_20_Most_Popular_Artists.Top_20_Most_Popular_Artist_Driver.java

/**
 * @param args the command line arguments
 *///from w  w  w .  jav  a  2  s  .  c om

public static void main(String[] args) throws IOException, InterruptedException, ClassNotFoundException {
    Configuration conf = new Configuration();
    Job job = Job.getInstance(conf, "Top 20 most popular artist ");
    job.setJarByClass(Top_20_Most_Popular_Artist_Driver.class);

    job.setMapperClass(Top_20_Most_Popular_Artist_Mapper.class);
    job.setMapOutputKeyClass(Text.class);
    job.setMapOutputValueClass(IntWritable.class);

    job.setNumReduceTasks(1);
    job.setCombinerClass(Top_20_Most_Popular_Artist_Combiner.class);
    job.setReducerClass(Top_20_Most_Popular_Artist_Reducer.class);
    job.setOutputKeyClass(NullWritable.class);
    job.setOutputValueClass(IntWritable.class);

    FileInputFormat.addInputPath(job, new Path(args[0]));
    FileOutputFormat.setOutputPath(job, new Path(args[1]));

    System.exit(job.waitForCompletion(true) ? 0 : 1);
}

---

From source file:Analysis.A3_Total_Users_By_Gender.User_Gender_Count_Driver.java

/**
 * @param args the command line arguments
 *///from   w w w. j  a v a  2  s.  c  om

public static void main(String[] args) throws IOException, InterruptedException, ClassNotFoundException {
    Configuration conf = new Configuration();
    Job job = Job.getInstance(conf, "Genderwise Demography on Service");
    job.setJarByClass(User_Gender_Count_Driver.class);
    job.setMapperClass(User_Gender_Count_Mapper.class);

    job.setCombinerClass(User_Gender_Count_Reducer.class);
    job.setReducerClass(User_Gender_Count_Reducer.class);

    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(IntWritable.class);

    FileInputFormat.addInputPath(job, new Path(args[0]));
    FileOutputFormat.setOutputPath(job, new Path(args[1]));
    System.exit(job.waitForCompletion(true) ? 0 : 1);
}

---

From source file:Analysis.A4_High_Traffic_Countries.Top_10_Countries_by_User_Traffic_Driver.java

/**
 * @param args the command line arguments
 *///w  w  w .  j av a  2  s  .c o  m

public static void main(String[] args) throws IOException, InterruptedException, ClassNotFoundException {
    Configuration conf = new Configuration();
    Job job = Job.getInstance(conf, "Top 10 Countries by User Traffic");
    job.setJarByClass(Top_10_Countries_by_User_Traffic_Driver.class);
    job.setMapperClass(Top_10_Countries_by_User_Traffic_Mapper.class);

    job.setCombinerClass(Top_10_Countries_by_User_Traffic_Combiner.class);
    job.setReducerClass(Top_10_Countries_by_User_Traffic_Reducer.class);

    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(IntWritable.class);

    FileInputFormat.addInputPath(job, new Path(args[0]));
    FileOutputFormat.setOutputPath(job, new Path(args[1]));
    System.exit(job.waitForCompletion(true) ? 0 : 1);
}

---

From source file:Analysis.A5_Min_Max_Median_Age_Top_Countries.Min_Max_Age_By_Country_Driver.java

/**
 * @param args the command line arguments
 *///from   w w w.  ja v  a2  s  .c om
public static void main(String[] args) throws IOException, InterruptedException, ClassNotFoundException {
    Configuration conf = new Configuration();
    Job job = Job.getInstance(conf, "Min, Max, Average, Median age of users by country");
    job.setJarByClass(Min_Max_Age_By_Country_Driver.class);
    job.setMapperClass(Min_Max_Age_By_Country_Mapper.class);

    job.setMapOutputKeyClass(Text.class);
    job.setMapOutputValueClass(IntWritable.class);

    job.setReducerClass(Min_Max_Age_By_Country_Reducer.class);

    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(Text.class);

    job.setNumReduceTasks(1);

    FileInputFormat.addInputPath(job, new Path(args[0]));
    FileOutputFormat.setOutputPath(job, new Path(args[1]));
    System.exit(job.waitForCompletion(true) ? 0 : 1);
}

---

From source file:Analysis.A6_User_Differentiation_By_Age.Partition_Users_By_Age_Driver.java

/**
 * @param args the command line arguments
 *//* w  ww . j a  va  2  s.c o  m*/
public static void main(String[] args) throws IOException, InterruptedException, ClassNotFoundException {
    Configuration conf = new Configuration();
    Job job = Job.getInstance(conf, "Users by Age");
    job.setJarByClass(Partition_Users_By_Age_Driver.class);

    job.setMapperClass(Partition_Users_By_Age_Mapper.class);
    job.setMapOutputKeyClass(IntWritable.class);
    job.setMapOutputValueClass(Text.class);

    // partitioner class inclusion
    job.setPartitionerClass(Partition_Users_By_Age_Partitioner.class);

    // set multiple formats for custom naming partitioning
    MultipleOutputs.addNamedOutput(job, "ageBins", TextOutputFormat.class, Text.class, NullWritable.class);
    MultipleOutputs.setCountersEnabled(job, true);

    //11-17, 18-25, 26-35, 36-49,50-65,66-80, 81-99

    // set num of reduce tasks based on partition we need (here we need 10 cos total no.of countries)
    job.setNumReduceTasks(8);
    job.setReducerClass(Partition_Users_By_Age_Reducer.class);

    FileInputFormat.addInputPath(job, new Path(args[0]));
    FileOutputFormat.setOutputPath(job, new Path(args[1]));
    System.exit(job.waitForCompletion(true) ? 0 : 1);
}

---

From source file:Analysis.A7_Total_Signups_By_Year.Total_Signup_by_Year_Driver.java

/**
 * @param args the command line arguments
 *///from   w  ww.  jav a2  s.  c  om

public static void main(String[] args) throws IOException, InterruptedException, ClassNotFoundException {
    Configuration conf = new Configuration();
    Job job = Job.getInstance(conf, "Total Signups by Year");
    job.setJarByClass(Total_Signup_by_Year_Driver.class);
    job.setMapperClass(Total_Signup_by_Year_Mapper.class);

    job.setCombinerClass(Total_Signup_by_Year_Reducer.class);
    job.setReducerClass(Total_Signup_by_Year_Reducer.class);

    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(IntWritable.class);

    FileInputFormat.addInputPath(job, new Path(args[0]));
    FileOutputFormat.setOutputPath(job, new Path(args[1]));
    System.exit(job.waitForCompletion(true) ? 0 : 1);
}

---

From source file:Analysis.A8_Top_10_Most_Popular_Tracks.Top_10_Most_Popular_Tracks_Driver.java

/**
 * @param args the command line arguments
 *///from  w  w w  .  j  av a2 s .c  om

public static void main(String[] args) throws IOException, InterruptedException, ClassNotFoundException {
    Configuration conf = new Configuration();
    Job job = Job.getInstance(conf, "Top 10 most popular tracks ");
    job.setJarByClass(Top_10_Most_Popular_Tracks_Driver.class);

    job.setMapperClass(Top_10_Most_Popular_Tracks_Mapper.class);
    job.setMapOutputKeyClass(Text.class);
    job.setMapOutputValueClass(IntWritable.class);

    job.setNumReduceTasks(1);
    job.setReducerClass(Top_10_Most_Popular_Tracks_Reducer.class);
    job.setOutputKeyClass(NullWritable.class);
    job.setOutputValueClass(IntWritable.class);

    FileInputFormat.addInputPath(job, new Path(args[0]));
    FileOutputFormat.setOutputPath(job, new Path(args[1]));

    System.exit(job.waitForCompletion(true) ? 0 : 1);
}

---

From source file:Analysis.A9_Max_Activity_By_Time_of_Day.Most_Listens_By_Time_of_Day_Driver.java

/**
 * @param args the command line arguments
 */// ww w  .j a  va2s .  c  o m
public static void main(String[] args) throws IOException, InterruptedException, ClassNotFoundException {
    Configuration conf = new Configuration();
    Job job = Job.getInstance(conf, "Most listens by Time of the Day");
    job.setJarByClass(Most_Listens_By_Time_of_Day_Driver.class);

    job.setMapperClass(Most_Listens_By_Time_of_Day_Mapper.class);
    job.setMapOutputKeyClass(Text.class);
    job.setMapOutputValueClass(NullWritable.class);

    FileInputFormat.addInputPath(job, new Path(args[0]));
    FileOutputFormat.setOutputPath(job, new Path(args[1]));

    int code = job.waitForCompletion(true) ? 0 : 1;

    if (code == 0) {
        for (Counter counter : job.getCounters()
                .getGroup(Most_Listens_By_Time_of_Day_Mapper.HOUR_COUNTER_GROUP)) {
            System.out.println(counter.getDisplayName() + "\t" + counter.getValue());
        }
    }

    FileSystem.get(conf).delete(new Path(args[1]), true);

    System.exit(code);
}

---

From source file:apex.benchmark.RedisHelper.java

License:Apache License

public void fillDB(String fileName) throws IOException {
    Path filePath = new Path(fileName);
    Configuration configuration = new Configuration();
    FileSystem fs;//from   w  ww. j av a2  s  . c  om
    fs = FileSystem.newInstance(filePath.toUri(), configuration);
    FSDataInputStream inputStream = fs.open(filePath);
    BufferedReader bufferedReader;

    try {

        bufferedReader = new BufferedReader(new InputStreamReader(inputStream));

        String line;
        while ((line = bufferedReader.readLine()) != null) {

            String[] mapping = line.split("\\s+");

            if (mapping.length != 2) {
                continue;
            }

            jedis.sadd("campaigns", mapping[0]);
            jedis.set(mapping[1], mapping[0]);
        }
    } catch (Exception e) {
        throw e;
    }
}

---

From source file:application.RecommenderEvaluator.java

License:Open Source License

/**
 *    il metodo evaluate prende in ingresso una collezione di item e media e standard error ad esso associato, 
 *    una collezione di user che ha come valore una collezione delle medie degli item per i quali ha espresso un voto,
 *    una stringa che contiene il path ad un file di testo (#user,#item,#vote) per effettuare i test. Il metodo fornisce in output la matrice di confusione cosi' formata 
 *    nella cella [0][0] ci sara' il numero di volte in cui il voto e' stato positivo e la previsione e' stata positiva (True Positive)
 *    nella cella [0][1] ci sara' il numero di volte in cui il voto e' stato positivo e la previsione e' stata negativa (False Negative)
 *    nella cella [1][0] ci sara' il numero di volte in cui il voto e' stato negativo e la previsione e' stata positiva (False Positive)
 *    nella cella [1][1] ci sara' il numero di volte in cui il voto e' stato negativo e la previsione e' stata negativa (True Negative)
 *    @param Stato_Item e' una collezione di item con media e standard error a ciascuno di esso associato
 *    @param Stato_User e' una collezione di user con valore una collezione delle medie degli item per i quali l'user ha espresso un voto
 *    @param s e' una stringa che contiene il path ad un file di testo per effettuare i test
 *    @return una matrice di confusione con il numero di TruePositive, FalsePositive, FalseNegative, TrueNegative ottenuti.
 *    @throws IOException// w w w .  j a va  2s  .  c  o m
 */
public void evaluate() {
    String test_file = GLOBALS.getTEST_FILE_NAME();
    String split = GLOBALS.getSPLIT_TOKEN();
    try {
        FileSystem fs = FileSystem.get(conf);
        BufferedReader br = new BufferedReader(new InputStreamReader(fs.open(new Path(test_file))));
        String line;
        while ((line = br.readLine()) != null) {
            StringTokenizer st = new StringTokenizer(line, split);
            int user = Integer.parseInt(st.nextToken());
            int item = Integer.parseInt(st.nextToken());
            int vote = Integer.parseInt(st.nextToken());
            UserProfile UP = USER_STATE.get(user);
            ItemProfile IP = ITEM_STATE.get(item);
            if (UP == null || IP == null) {
                skipped_evaluation++;
            } else {

                Boolean prediction = estimatePreference(UP, IP);
                if (prediction == null) {
                    skipped_evaluation++;
                    continue;
                }

                //               System.out.println("prediction:"+prediction+", vote:"+vote);
                if (prediction) {
                    if (vote == 1) {
                        CM[0][0] += 1;
                    } //TP
                    else if (vote == -1) {
                        CM[1][0] += 1;
                    } //FP
                } else {
                    if (vote == -1) {
                        CM[1][1] += 1;
                    } //TN
                    else if (vote == 1) {
                        CM[0][1] += 1;
                    } //FN
                }
            }
        }
        br.close();

        //STAT
        //         System.out.println("TP: "+CM[0][0]);
        //         System.out.println("FN: "+CM[0][1]);
        //         System.out.println("FP: "+CM[1][0]);
        //         System.out.println("TN: "+CM[1][1]);
        //         System.out.println("SKYPPED: "+skipped_evaluation);
    } catch (Exception e) {
        System.out.println("Exception " + e);
        e.printStackTrace();
        System.exit(-1);
    }
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• 37
• 38
• 39
• 40
• 41
• 42
• 43
• 44
• 45
• 46
• 47
• »