yaravind · May 18, 2018 02:41 · Jul 3, 2013 · Jul 3, 2013 · Jul 3, 2013 · Jul 3, 2013
diff --git a/04-HiveCommands b/04-HiveCommands
@@ -18,7 +18,8 @@ WITH SERDEPROPERTIES (
 stored as textfile;
 
 b) Create partitions and load data:
-[Replace '/user/airawat' with '/user/<your userID>']
+
+Note: Replace '/user/airawat' with '/user/<your userID>'
 
 hive> Alter table LogParserSample Add IF NOT EXISTS partition(year=2013, month=04)
      location '/user/airawat/LogParserSampleHive/logs/airawat-syslog/2013/04/';

diff --git a/05-QueryAndOutput b/05-QueryAndOutput
@@ -5,21 +5,6 @@ hive> set hive.cli.print.header=true;
 hive> add jar hadoop-lib/hive-contrib-0.10.0-cdh4.2.0.jar; --I need this as my environment is not properly configured
 hive> select Year,Month,Day,Event,Count(*) Occurrence from LogParserSample group by year,month,day,event order by event desc,year,month,day;
 
-Output
--------
-year  month	day	event	occurrence
-2013	05	7	udevd[361]:	1
-2013	04	23	sudo:	1
-2013	05	3	sudo:	1
-2013	05	3	ntpd_initres[1705]:	144
-2013	05	4	ntpd_initres[1705]:	261
-2013	05	5	ntpd_initres[1705]:	264
-2013	05	6	ntpd_initres[1705]:	123
-2013	05	3	kernel:	5
-2013	05	6	kernel:	1
-2013	05	7	kernel:	52
-2013	05	3	init:	5
-2013	05	7	init:	18
 
 
 

diff --git a/06-HiveQueryOutput b/06-HiveQueryOutput
@@ -0,0 +1,16 @@
+Query output
+------------
+
+year  month  day	event	occurrence
+2013	05	7	udevd[361]:	1
+2013	04	23	sudo:	1
+2013	05	3	sudo:	1
+2013	05	3	ntpd_initres[1705]:	144
+2013	05	4	ntpd_initres[1705]:	261
+2013	05	5	ntpd_initres[1705]:	264
+2013	05	6	ntpd_initres[1705]:	123
+2013	05	3	kernel:	5
+2013	05	6	kernel:	1
+2013	05	7	kernel:	52
+2013	05	3	init:	5
+2013	05	7	init:	18
diff --git a/05-QueryAndOutput b/05-QueryAndOutput
@@ -7,6 +7,21 @@ hive> select Year,Month,Day,Event,Count(*) Occurrence from LogParserSample group
 
 Output
 -------
+year  month	day	event	occurrence
+2013	05	7	udevd[361]:	1
+2013	04	23	sudo:	1
+2013	05	3	sudo:	1
+2013	05	3	ntpd_initres[1705]:	144
+2013	05	4	ntpd_initres[1705]:	261
+2013	05	5	ntpd_initres[1705]:	264
+2013	05	6	ntpd_initres[1705]:	123
+2013	05	3	kernel:	5
+2013	05	6	kernel:	1
+2013	05	7	kernel:	52
+2013	05	3	init:	5
+2013	05	7	init:	18
+
+
 
 
 

diff --git a/04-HiveCommands b/04-HiveCommands
@@ -18,6 +18,7 @@ WITH SERDEPROPERTIES (
 stored as textfile;
 
 b) Create partitions and load data:
+[Replace '/user/airawat' with '/user/<your userID>']
 
 hive> Alter table LogParserSample Add IF NOT EXISTS partition(year=2013, month=04)
      location '/user/airawat/LogParserSampleHive/logs/airawat-syslog/2013/04/';

diff --git a/02-DataDownload b/02-DataDownload
@@ -1,6 +1,16 @@
 Data download
 -------------
+https://groups.google.com/forum/?hl=en#!topic/hadooped/_tj8w_E-MGY
 
 
 Directory structure
--------------------
+-------------------
+LogParserSampleHive
+    logs
+        airawat-syslog
+            2013
+                04
+                    messages
+            2013
+                05
+                    messages
diff --git a/03-DataLoadCommands b/03-DataLoadCommands
@@ -1,2 +1,15 @@
 Data load commands
-------------------
+------------------
+a) Load the data
+$ hadoop fs -mkdir LogParserSampleHive
+$ hadoop fs -mkdir LogParserSampleHive/logs
+$ hadoop fs -put LogParserSampleHive/logs/* LogParserSampleHive/logs/
+$ hadoop fs -ls -R LogParserSampleHive/ | awk {'print $8'}
+
+LogParserSampleHive/logs
+LogParserSampleHive/logs/airawat-syslog
+LogParserSampleHive/logs/airawat-syslog/2013
+LogParserSampleHive/logs/airawat-syslog/2013/04
+LogParserSampleHive/logs/airawat-syslog/2013/04/messages
+LogParserSampleHive/logs/airawat-syslog/2013/05
+LogParserSampleHive/logs/airawat-syslog/2013/05/messages
diff --git a/04-HiveCommands b/04-HiveCommands
@@ -1,2 +1,26 @@
 Hive commands
---------------
+--------------
+
+a) Create external table:
+
+hive> CREATE EXTERNAL TABLE LogParserSample(
+month_name STRING,
+day STRING,
+time STRING,
+host STRING,
+event STRING,
+log STRING)
+PARTITIONED BY(year int, month int)
+ROW FORMAT SERDE 'org.apache.hadoop.hive.contrib.serde2.RegexSerDe' 
+WITH SERDEPROPERTIES (
+ "input.regex" = "(\\w+)\\s+(\\d+)\\s+(\\d+:\\d+:\\d+)\\s+(\\w+\\W*\\w*)\\s+(.*?\\:)\\s+(.*$)"
+) 
+stored as textfile;
+
+b) Create partitions and load data:
+
+hive> Alter table LogParserSample Add IF NOT EXISTS partition(year=2013, month=04)
+     location '/user/airawat/LogParserSampleHive/logs/airawat-syslog/2013/04/';
+
+hive> Alter table LogParserSample Add IF NOT EXISTS partition(year=2013, month=05)
+     location '/user/airawat/LogParserSampleHive/logs/airawat-syslog/2013/05/'; 
diff --git a/05-QueryAndOutput b/05-QueryAndOutput
@@ -0,0 +1,15 @@
+Hive query
+-----------
+
+hive> set hive.cli.print.header=true;
+hive> add jar hadoop-lib/hive-contrib-0.10.0-cdh4.2.0.jar; --I need this as my environment is not properly configured
+hive> select Year,Month,Day,Event,Count(*) Occurrence from LogParserSample group by year,month,day,event order by event desc,year,month,day;
+
+Output
+-------
+
+
+
+
+
+
diff --git a/05-SampleOutput b/05-SampleOutput
@@ -1,2 +0,0 @@
-Hive query and output
----------------------

diff --git a/00-LogParser-Hive-Regex b/00-LogParser-Hive-Regex
@@ -1,13 +1,12 @@
-This gist includes a mapper, reducer and driver in java that can parse log files using 
-regex; The code for combiner is the same as reducer;
-Usecase:  Count the number of occurances of processes that got logged, inception to date.
+This gist includes hive ql scripts to create an external partitioned table for Syslog
+generated log files using regex serde; 
+Usecase:  Count the number of occurances of processes that got logged, by year, month,
+day and process.
 
 Includes:
 ---------
-Mapper:                              01-LogEventCountMapper.java
-Reducer:                             02-LogEventCountReducer.java 
-Driver:                              03-LogEventCount.java 
-Sample data and scripts for download:04-ScriptAndDataDownload
-Sample data and structure:           05-SampleDataAndStructure 
-Commands:                            06-Commands
-Sample output:                       07-Output
+Sample data and structure:           01-SampleDataAndStructure 
+Data download:                       02-DataDownload
+Data load commands:                  03-DataLoadCommands
+Hive commands:                       04-HiveCommands
+Sample output:                       05-SampleOutput
diff --git a/01-SampleDataAndStructure b/01-SampleDataAndStructure
@@ -0,0 +1,18 @@
+Sample data
+------------
+May  3 11:52:54 cdh-dn03 init: tty (/dev/tty6) main process (1208) killed by TERM signal
+May  3 11:53:31 cdh-dn03 kernel: registered taskstats version 1
+May  3 11:53:31 cdh-dn03 kernel: sr0: scsi3-mmc drive: 32x/32x xa/form2 tray
+May  3 11:53:31 cdh-dn03 kernel: piix4_smbus 0000:00:07.0: SMBus base address uninitialized - upgrade BIOS or use force_addr=0xaddr
+May  3 11:53:31 cdh-dn03 kernel: nf_conntrack version 0.5.0 (7972 buckets, 31888 max)
+May  3 11:53:57 cdh-dn03 kernel: hrtimer: interrupt took 11250457 ns
+May  3 11:53:59 cdh-dn03 ntpd_initres[1705]: host name not found: 0.rhel.pool.ntp.org
+
+Structure
+----------
+Month   = May  
+Day     = 3 
+Time    = 11:52:54 
+Node    = cdh-dn03 
+Process = init: 
+Log msg = tty (/dev/tty6) main process (1208) killed by TERM signal
diff --git a/02-DataDownload b/02-DataDownload
@@ -0,0 +1,6 @@
+Data download
+-------------
+
+
+Directory structure
+-------------------
diff --git a/03-DataLoadCommands b/03-DataLoadCommands
@@ -0,0 +1,2 @@
+Data load commands
+------------------
diff --git a/04-HiveCommands b/04-HiveCommands
@@ -0,0 +1,2 @@
+Hive commands
+--------------
diff --git a/05-SampleOutput b/05-SampleOutput
@@ -0,0 +1,2 @@
+Hive query and output
+---------------------
diff --git a/00-LogParser-Hive-Regex b/00-LogParser-Hive-Regex
@@ -0,0 +1,13 @@
+This gist includes a mapper, reducer and driver in java that can parse log files using 
+regex; The code for combiner is the same as reducer;
+Usecase:  Count the number of occurances of processes that got logged, inception to date.
+
+Includes:
+---------
+Mapper:                              01-LogEventCountMapper.java
+Reducer:                             02-LogEventCountReducer.java 
+Driver:                              03-LogEventCount.java 
+Sample data and scripts for download:04-ScriptAndDataDownload
+Sample data and structure:           05-SampleDataAndStructure 
+Commands:                            06-Commands
+Sample output:                       07-Output
No results found