3 from ..parser import ApacheLogParserError, FORMATS, Parser
6 class TestApacheLogParser(unittest.TestCase):
9 self.format = r'%h %l %u %t \"%r\" %>s '\
10 r'%b \"%{Referer}i\" \"%{User-Agent}i\"'
11 self.fields = '%h %l %u %t %r %>s %b %{Referer}i '\
12 '%{User-Agent}i'.split(' ')
13 self.pattern = '^(\\S*) (\\S*) (\\S*) (\\[[^\\]]+\\]) '\
14 '\\\"([^"\\\\]*(?:\\\\.[^"\\\\]*)*)\\\" '\
15 '(\\S*) (\\S*) \\\"([^"\\\\]*(?:\\\\.[^"\\\\]*)*)\\\" '\
16 '\\\"([^"\\\\]*(?:\\\\.[^"\\\\]*)*)\\\"$'
17 self.line1 = r'212.74.15.68 - - [23/Jan/2004:11:36:20 +0000] '\
18 r'"GET /images/previous.png HTTP/1.1" 200 2607 '\
19 r'"http://peterhi.dyndns.org/bandwidth/index.html" '\
20 r'"Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.2) '\
22 self.line2 = r'212.74.15.68 - - [23/Jan/2004:11:36:20 +0000] '\
23 r'"GET /images/previous.png=\" HTTP/1.1" 200 2607 '\
24 r'"http://peterhi.dyndns.org/bandwidth/index.html" '\
25 r'"Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.2) '\
27 self.line3 = r'4.224.234.46 - - [20/Jul/2004:13:18:55 -0700] '\
28 r'"GET /core/listing/pl_boat_detail.jsp?&units=Feet&checked'\
29 r'_boats=1176818&slim=broker&&hosturl=giffordmarine&&ywo='\
30 r'giffordmarine& HTTP/1.1" 200 2888 "http://search.yahoo.com/'\
31 r'bin/search?p=\"grady%20white%20306%20bimini\"" '\
32 r'"\"Mozilla/4.0 (compatible; MSIE 6.0; Windows 98; '\
33 r'YPC 3.0.3; yplus 4.0.00d)\""'
34 # r'"Mozilla/4.0 (compatible; MSIE 6.0; Windows 98; '\
35 # r'YPC 3.0.3; yplus 4.0.00d)"'
36 self.p = Parser(self.format)
38 def testpattern(self):
39 self.assertEqual(self.pattern, self.p.pattern())
42 self.assertEqual(self.fields, self.p.names())
45 data = self.p.parse(self.line1)
46 self.assertEqual(data['%h'], '212.74.15.68', msg = 'Line 1 %h')
47 self.assertEqual(data['%l'], '-', msg = 'Line 1 %l')
48 self.assertEqual(data['%u'], '-', msg = 'Line 1 %u')
49 self.assertEqual(data['%t'], '[23/Jan/2004:11:36:20 +0000]', msg = 'Line 1 %t')
52 'GET /images/previous.png HTTP/1.1',
55 self.assertEqual(data['%>s'], '200', msg = 'Line 1 %>s')
56 self.assertEqual(data['%b'], '2607', msg = 'Line 1 %b')
59 'http://peterhi.dyndns.org/bandwidth/index.html',
60 msg = 'Line 1 %{Referer}i'
63 data['%{User-Agent}i'],
64 'Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.2) Gecko/20021202',
65 msg = 'Line 1 %{User-Agent}i'
70 data = self.p.parse(self.line2)
71 self.assertEqual(data['%h'], '212.74.15.68', msg = 'Line 2 %h')
72 self.assertEqual(data['%l'], '-', msg = 'Line 2 %l')
73 self.assertEqual(data['%u'], '-', msg = 'Line 2 %u')
76 '[23/Jan/2004:11:36:20 +0000]',
81 r'GET /images/previous.png=\" HTTP/1.1',
84 self.assertEqual(data['%>s'], '200', msg = 'Line 2 %>s')
85 self.assertEqual(data['%b'], '2607', msg = 'Line 2 %b')
88 'http://peterhi.dyndns.org/bandwidth/index.html',
89 msg = 'Line 2 %{Referer}i'
92 data['%{User-Agent}i'],
93 'Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.2) Gecko/20021202',
94 msg = 'Line 2 %{User-Agent}i'
98 data = self.p.parse(self.line3)
99 self.assertEqual(data['%h'], '4.224.234.46', msg = 'Line 3 %h')
100 self.assertEqual(data['%l'], '-', msg = 'Line 3 %l')
101 self.assertEqual(data['%u'], '-', msg = 'Line 3 %u')
104 '[20/Jul/2004:13:18:55 -0700]',
109 r'GET /core/listing/pl_boat_detail.jsp?&units=Feet&checked_boats='\
110 r'1176818&slim=broker&&hosturl=giffordmarine&&ywo=giffordmarine& '\
114 self.assertEqual(data['%>s'], '200', msg = 'Line 3 %>s')
115 self.assertEqual(data['%b'], '2888', msg = 'Line 3 %b')
118 r'http://search.yahoo.com/bin/search?p=\"grady%20white%20306'\
120 msg = 'Line 3 %{Referer}i'
123 data['%{User-Agent}i'],
124 '\\"Mozilla/4.0 (compatible; MSIE 6.0; Windows 98; YPC 3.0.3; '\
126 # 'Mozilla/4.0 (compatible; MSIE 6.0; Windows 98; YPC 3.0.3; '\
128 msg = 'Line 3 %{User-Agent}i'
132 def testjunkline(self):
133 self.assertRaises(ApacheLogParserError,self.p.parse,'foobar')
135 def testhasquotesaltn(self):
136 p = Parser(r'%a \"%b\" %c')
137 line = r'foo "xyz" bar'
139 self.assertEqual(data['%a'],'foo', '%a')
140 self.assertEqual(data['%b'],'xyz', '%c')
141 self.assertEqual(data['%c'],'bar', '%c')
143 class TestApacheLogParserFriendlyNames(unittest.TestCase):
146 self.format = r'%h %l %u %t \"%r\" %>s '\
147 r'%b \"%{Referer}i\" \"%{User-Agent}i\"'
148 self.fields = ('remote_host remote_logname remote_user time '
149 'first_line last_status response_bytes_clf '
150 'header_Referer header_User_Agent').split(' ')
151 self.pattern = '^(\\S*) (\\S*) (\\S*) (\\[[^\\]]+\\]) '\
152 '\\\"([^"\\\\]*(?:\\\\.[^"\\\\]*)*)\\\" '\
153 '(\\S*) (\\S*) \\\"([^"\\\\]*(?:\\\\.[^"\\\\]*)*)\\\" '\
154 '\\\"([^"\\\\]*(?:\\\\.[^"\\\\]*)*)\\\"$'
155 self.line1 = r'212.74.15.68 - - [23/Jan/2004:11:36:20 +0000] '\
156 r'"GET /images/previous.png HTTP/1.1" 200 2607 '\
157 r'"http://peterhi.dyndns.org/bandwidth/index.html" '\
158 r'"Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.2) '\
160 self.line2 = r'212.74.15.68 - - [23/Jan/2004:11:36:20 +0000] '\
161 r'"GET /images/previous.png=\" HTTP/1.1" 200 2607 '\
162 r'"http://peterhi.dyndns.org/bandwidth/index.html" '\
163 r'"Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.2) '\
165 self.line3 = r'4.224.234.46 - - [20/Jul/2004:13:18:55 -0700] '\
166 r'"GET /core/listing/pl_boat_detail.jsp?&units=Feet&checked'\
167 r'_boats=1176818&slim=broker&&hosturl=giffordmarine&&ywo='\
168 r'giffordmarine& HTTP/1.1" 200 2888 "http://search.yahoo.com/'\
169 r'bin/search?p=\"grady%20white%20306%20bimini\"" '\
170 r'"\"Mozilla/4.0 (compatible; MSIE 6.0; Windows 98; '\
171 r'YPC 3.0.3; yplus 4.0.00d)\""'
172 # r'"Mozilla/4.0 (compatible; MSIE 6.0; Windows 98; '\
173 # r'YPC 3.0.3; yplus 4.0.00d)"'
174 self.p = Parser(self.format, True)
176 def testpattern(self):
177 self.assertEqual(self.pattern, self.p.pattern())
180 self.assertEqual(self.fields, self.p.names())
183 data = self.p.parse(self.line1)
184 self.assertEqual(data.remote_host, '212.74.15.68', msg = 'Line 1 remote_host')
185 self.assertEqual(data.remote_logname, '-', msg = 'Line 1 remote_logname')
186 self.assertEqual(data.remote_user, '-', msg = 'Line 1 remote_user')
187 self.assertEqual(data.time, '[23/Jan/2004:11:36:20 +0000]', msg = 'Line 1 time')
190 'GET /images/previous.png HTTP/1.1',
191 msg = 'Line 1 first_line'
193 self.assertEqual(data.last_status, '200', msg = 'Line 1 last_status')
194 self.assertEqual(data.response_bytes_clf, '2607', msg = 'Line 1 response_bytes_clf')
197 'http://peterhi.dyndns.org/bandwidth/index.html',
198 msg = 'Line 1 %{Referer}i'
201 data.header_User_Agent,
202 'Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.2) Gecko/20021202',
203 msg = 'Line 1 %{User-Agent}i'
208 data = self.p.parse(self.line2)
209 self.assertEqual(data.remote_host, '212.74.15.68', msg = 'Line 2 remote_host')
210 self.assertEqual(data.remote_logname, '-', msg = 'Line 2 remote_logname')
211 self.assertEqual(data.remote_user, '-', msg = 'Line 2 remote_user')
214 '[23/Jan/2004:11:36:20 +0000]',
219 r'GET /images/previous.png=\" HTTP/1.1',
220 msg = 'Line 2 first_line'
222 self.assertEqual(data.last_status, '200', msg = 'Line 2 last_status')
223 self.assertEqual(data.response_bytes_clf, '2607', msg = 'Line 2 response_bytes_clf')
226 'http://peterhi.dyndns.org/bandwidth/index.html',
227 msg = 'Line 2 %{Referer}i'
230 data.header_User_Agent,
231 'Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.2) Gecko/20021202',
232 msg = 'Line 2 %{User-Agent}i'
236 data = self.p.parse(self.line3)
237 self.assertEqual(data.remote_host, '4.224.234.46', msg = 'Line 3 remote_host')
238 self.assertEqual(data.remote_logname, '-', msg = 'Line 3 remote_logname')
239 self.assertEqual(data.remote_user, '-', msg = 'Line 3 remote_user')
242 '[20/Jul/2004:13:18:55 -0700]',
247 r'GET /core/listing/pl_boat_detail.jsp?&units=Feet&checked_boats='\
248 r'1176818&slim=broker&&hosturl=giffordmarine&&ywo=giffordmarine& '\
250 msg = 'Line 3 first_line'
252 self.assertEqual(data.last_status, '200', msg = 'Line 3 last_status')
253 self.assertEqual(data.response_bytes_clf, '2888', msg = 'Line 3 response_bytes_clf')
256 r'http://search.yahoo.com/bin/search?p=\"grady%20white%20306'\
258 msg = 'Line 3 %{Referer}i'
261 data.header_User_Agent,
262 '\\"Mozilla/4.0 (compatible; MSIE 6.0; Windows 98; YPC 3.0.3; '\
264 # 'Mozilla/4.0 (compatible; MSIE 6.0; Windows 98; YPC 3.0.3; '\
266 msg = 'Line 3 %{User-Agent}i'
270 def testjunkline(self):
271 self.assertRaises(ApacheLogParserError,self.p.parse,'foobar')
273 def testhasquotesaltn(self):
274 p = Parser(r'%a \"%b\" %c')
275 line = r'foo "xyz" bar'
277 self.assertEqual(data['%a'],'foo', '%a')
278 self.assertEqual(data['%b'],'xyz', '%c')
279 self.assertEqual(data['%c'],'bar', '%c')
282 if __name__ is '__main__':