-
Notifications
You must be signed in to change notification settings - Fork 0
/
assign1.java
136 lines (127 loc) · 4.28 KB
/
assign1.java
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
/*
Authors:
Divya Gupta B17083
Komal Mahale B17089
Piyush Bafna B17020
To run the hadoop program use
1) javac -cp /usr/local/hadoopUser/hadoop-2.9.0/share/hadoop/mapreduce/hadoop-mapreduce-client-core-2.9.0.jar:/usr/local/hadoopUser/hadoop-2.9.0/share/hadoop/common/hadoop-common-2.9.0.jar:/usr/local/hadoopUser/hadoop-2.9.0/share/hadoop/common/lib/commons-cli-1.2.jar -d /Users/bhawesh/Desktop/divya/assign_db/classfolder /Users/bhawesh/Desktop/divya/assign_db/assign1.java
2) jar -cvf /Users/bhawesh/Desktop/divya/assign_db/assign1.jar -C /Users/bhawesh/Desktop/divya/assign_db/classfolder .
3) bin/hadoop jar /Users/bhawesh/Desktop/divya/assign_db/assign1.jar assign1 /user/divya/assign1inp/in.txt /user/divya/output
*/
import java.io.*;
import org.apache.hadoop.io.*;
import org.apache.hadoop.mapreduce.*;
import org.apache.hadoop.conf.*;
import org.apache.hadoop.conf.*;
import org.apache.hadoop.fs.*;
import org.apache.hadoop.mapreduce.lib.input.*;
import org.apache.hadoop.mapreduce.lib.output.*;
import org.apache.hadoop.util.*;
public class assign1
{
//Map class
public static class MapClass extends Mapper<LongWritable,Text,Text,Text>
{
public void map(LongWritable key, Text value, Context context)
{
try{
String[] str = value.toString().split(" ");
String tid=str[0];
context.write(new Text(tid), new Text(value));
}
catch(Exception e)
{
System.out.println(e.getMessage());
}
}
}
//Reducer class
public static class ReduceClass extends Reducer<Text,Text,Text,Text>
{
public void reduce(Text key, Iterable <Text> values, Context context) throws IOException, InterruptedException
{
int total_cost=0;
int no_items=0;
String tid = key.toString();
for (Text val : values)
{
String [] str = val.toString().split(" ", -3);
no_items= str.length-1;
for(int i=1;i<str.length;i++)
total_cost = total_cost + Integer.parseInt(str[i]);
}
String result = tid+">"+String.valueOf(total_cost)+","+String.valueOf(no_items);
if(no_items>=1 & no_items<=10)
{
tid="bin1";
}
else if(no_items>=11 & no_items<=20)
{
tid="bin2";
}
else if(no_items>=21 & no_items<=30)
{
tid="bin3";
}
else if(no_items>=31 & no_items<=40)
{
tid="bin4";
}
else if(no_items>=41 & no_items<=50)
{
tid="bin5";
}
context.write(new Text(tid), new Text(result));
}
}
//partition class
public static class Partitionerclass extends
Partitioner < Text, Text >
{
@Override
public int getPartition(Text key, Text value, int numReduceTasks)
{
String[] str = value.toString().split(" ");
int no_items = str.length-1;
if(no_items>=1 & no_items<=10)
{
return 0;
}
else if(no_items>=11 & no_items<=20)
{
return 1 % numReduceTasks;
}
else if(no_items>=21 & no_items<=30)
{
return 2 % numReduceTasks;
}
else if(no_items>=31 & no_items<=40)
{
return 3 % numReduceTasks;
}
else
{
return 4 % numReduceTasks;
}
}
}
public static void main(String[] args) throws Exception
{
Configuration conf = new Configuration();
Job job = Job.getInstance(conf, "word count");
job.setJarByClass(assign1.class);
FileInputFormat.addInputPath(job, new Path(args[0]));
FileOutputFormat.setOutputPath(job, new Path(args[1]));
job.setMapperClass(MapClass.class);
job.setNumReduceTasks(5);
job.setMapOutputKeyClass(Text.class);
job.setMapOutputValueClass(Text.class);
job.setPartitionerClass(Partitionerclass.class);
job.setReducerClass(ReduceClass.class);
job.setInputFormatClass(TextInputFormat.class);
job.setOutputFormatClass(TextOutputFormat.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(Text.class);
System.exit(job.waitForCompletion(true)? 0 : 1);
}
}