class Tag:
name = '';
text = '';
first_child = 0;
parent = 0;
next_sibling = 0;
closed = 0;
depth = 0;
def get_tag_info_str(self):
c,p,s = 'none','none','none'
if self.first_child != 0:
c = self.first_child.name
if self.parent != 0:
p = self.parent.name
if self.next_sibling != 0:
s = self.next_sibling.name
return "name = {}, text = {}\nParent = {}, First Child = {}, Next Sibling = {}\nClosed = {}, Depth = {}\n".format(self.name, self.text, p, c, s, self.closed, self.depth)
from HTMLParser import HTMLParser
from htmlentitydefs import name2codepoint
class MyHTMLParser(HTMLParser):
tag_list = []
depth = 0;
previous_tag = 'none';
mode = 'silent';
def handle_starttag(self, tag, attrs):
if self.mode != 'silent':
print "Start tag:", tag
for attr in attrs:
print " attr:", attr
self.depth = self.depth + 1
t = Tag()
t.name = tag
t.depth = self.depth
if self.previous_tag == 'start':
# current tag is a first child of the last tag
t.parent = self.tag_list[len(self.tag_list)-1]
self.tag_list[len(self.tag_list)-1].first_child = t
elif self.previous_tag == 'end':
# current tag is next sibling of the last tag
for x in reversed(self.tag_list):
if x.depth == self.depth:
x.next_sibling = t
if t.parent == 0:
t.parent = x.parent
break
elif self.previous_tag == 'startend':
# current tag is the next sibling of the previous tag
t.parent = self.tag_list[len(self.tag_list)-1].parent
self.tag_list[len(self.tag_list)-1].next_sibling = t
self.tag_list.append(t)
self.previous_tag = 'start'
def handle_endtag(self, tag):
if self.mode != 'silent':
print "End tag :", tag
for x in reversed(self.tag_list):
if x.name == tag and x.closed == 0:
x.closed = 1
break
self.depth = self.depth - 1
self.previous_tag = 'end'
def handle_startendtag(self, tag, attrs):
if self.mode != 'silent':
print "Start/End tag :", tag
for attr in attrs:
print " attr:", attr
t = Tag()
self.depth = self.depth + 1
t.name = tag
t.depth = self.depth
t.closed = 1
if self.previous_tag == 'start':
# current tag is first child of the last tag
t.parent = self.tag_list[len(self.tag_list)-1]
self.tag_list[len(self.tag_list)-1].first_child = t
elif self.previous_tag == 'startend':
# current tag is next sibling of last tag
t.parent = self.tag_list[len(self.tag_list)-1].parent
self.tag_list[len(self.tag_list)-1].next_sibling = t
elif self.previous_tag == 'end':
# current tag is next sibling of a previous tag of depth = self.depth
for x in reversed(self.tag_list):
if x.depth == self.depth:
x.next_sibling = t
if t.parent == 0:
t.parent = x.parent
break
self.tag_list.append(t)
self.depth = self.depth - 1
self.previous_tag = 'startend'
def handle_data(self, data):
if self.mode != 'silent':
print "Data :", data
self.depth = self.depth + 1
# add data to last tag in list with depth = current depth - 1
for x in reversed(self.tag_list):
if x.depth == self.depth - 1:
x.text = (x.text + ' ' + data.strip(' \n\t')).strip(' \n\t')
break
self.depth = self.depth - 1
def handle_comment(self, data):
if self.mode != 'silent':
print "Comment :", data
def handle_entityref(self, name):
if self.mode != 'silent':
c = unichr(name2codepoint[name])
print "Named ent:", c
def handle_charref(self, name):
if self.mode != 'silent':
if name.startswith('x'):
c = unichr(int(name[1:], 16))
else:
c = unichr(int(name))
print "Num ent :", c
def handle_decl(self, data):
if self.mode != 'silent':
print "Decl :", data
def print_tag_list(self, u):
for l in self.tag_list:
print l.get_tag_info_str()
def clear_tag_list(self):
self.tag_list.__delslice__(0,len(self.tag_list))
def pretty_print_tags(self):
for t in self.tag_list:
s = ''
s = s + self.get_indent_str(t.depth-1)
s = s + self.get_tag_str(t.name)
print s
def get_indent_str(self, n):
s = ''
while(n != 0):
s = s + ' '
n = n - 1
return s
def get_tag_str(self, name):
return '<{}>'.format(name)
def find_first_tag(self, name):
r = 0
for t in self.tag_list:
if t.name == name:
r = t
break
return r
def print_first_tag_info(self, name):
t = self.find_first_tag(name)
if t == 0:
print "Tag: {} not found".format(name)
else:
print t.get_tag_info_str()
import urllib
import socket
socket.setdefaulttimeout(10)
import httplib
class WebCrawler:
"""A simple web crawler"""
link_dict = {};
initial_depth = 0;
#filter_list = [];
parser = 0;
re_compiled_obj = 0;
class PageInfo:
""" i store info about a webpage here """
has_been_scraped = 0;
word_dict = {};
def __init__(self,re_compiled_obj):
#self.filter_list.append(self.Filter(1,'.cnn.'))
self.parser = MyHTMLParser()
self.re_compiled_obj = re_compiled_obj
def get_page(self,url):
""" loads a webpage into a string """
page = ''
try:
f = urllib.urlopen(url=url)
page = f.read()
f.close()
except IOError:
print "Error opening {}".format(url)
except httplib.InvalidURL, e:
print "{} caused an Invalid URL error.".format(url)
if hasattr(e, 'reason'):
print 'We failed to reach a server.'
print 'Reason: ', e.reason
elif hasattr(e, 'code'):
print 'The server couldn\'t fulfill the request.'
print 'Error code: ', e.code
return page
def check_filters(self,url):
""" If the url_str matches any of the
enabled filter strings
then put the url in the dictionary """
match = self.re_compiled_obj.search(url)
#print "match = {}".format(match)
return match
def find_h1_tag(self,s,pos):
""" finds the first <h1> tag """
start = s.find('<h1>',pos)
end = s.find('</h1>',start)
return start, end
def save_tag_text(self, tag, d):
""" stores each word in the tag in a dictionary """
if tag != 0:
token_list = tag.text.split(' ')
for token in token_list:
#print 'token = {}'.format(token)
if d.has_key(token):
d[token] = d[token] + 1
else:
d[token] = 1
return d
def save_page_text(self,page_str):
""" Save all important text on the page """
offset = 0
d = {}
while offset != -1:
start,end = self.find_h1_tag(page_str,offset)
offset = end
if start != -1 and end != -1:
h1_tag = page_str[start:end+5]
#print h1_tag
self.parser.clear_tag_list()
# turn text into linked list of tags
# only feed part of the page into the parser
self.parser.feed(h1_tag)
#self.parser.pretty_print_tags()
tag = self.parser.find_first_tag('h1')
# add words from tag into the dictionary
d = self.save_tag_text(tag,d)
return d
def save_all_links_on_page(self,page_str,limit=60):
""" Stores all links found on the current page in a dictionary """
d = {}
offset = 0
i = 0
num_pages_filtered = 0
num_duplicate_pages = 0
while offset != -1:
if i == limit:
break
offset = page_str.find('<a href="http',offset)
if offset != -1:
start = page_str.find('"', offset)
end = page_str.find('"',start+1)
link = page_str[start+1:end]
# don't just save all the links
# filter the links that match specified criteria
if self.check_filters(link):
if link not in self.link_dict:
# adding link to global dictionary
self.link_dict[link] = self.PageInfo()
# adding link to local dictionary
d[link] = self.PageInfo()
else:
num_duplicate_pages = num_duplicate_pages + 1
else:
num_pages_filtered = num_pages_filtered + 1
offset = offset + 1
i = i + 1
print "{} out of {} links were filtered".format(num_pages_filtered,i)
print "{} out of {} links were duplicates".format(num_duplicate_pages,i)
#print "{} links are being returned from save_all_links".format(len(d))
return d
def save_all_links_recursive(self,links,depth):
""" Recursive function that
1) converts each page (link) into a string
2) stores all links found in a dictionary """
d = {}
print "We are {} levels deep".format(self.initial_depth - depth)
if depth != 0:
depth = depth - 1
urls = links.viewkeys()
#print "There are {} urls".format(len(urls))
for url in urls:
print "trying to get {} over the internet".format(url)
page_str = self.get_page(url)
print "done getting {} over the internet".format(url)
self.link_dict[url].word_dict = self.save_page_text(page_str)
d = self.save_all_links_on_page(page_str)
self.link_dict[url].has_been_scraped = 1
# d contains all the links found on the current page
self.save_all_links_recursive(d,depth)
def start_crawling(self,seed_pages,depth):
""" User calls this function to start crawling the web """
d = {}
self.link_dict.clear()
# initialize global dictionary variable to the seed page url's passed in
for page in seed_pages:
self.link_dict[page] = self.PageInfo()
d[page] = self.PageInfo()
self.initial_depth = depth
# start a recursive crawl
# can't pass in self.link_dict because then i get a RuntimeError: dictionary changed size during iteration
self.save_all_links_recursive(d,depth)
def print_all_page_text(self):
""" prints contents of all the word dictionaries """
for i in range(len(self.link_dict)):
page_info = self.link_dict.values()[i]
url = self.link_dict.keys()[i]
print 'url = {}, has_been_scraped = {}'.format(url,page_info.has_been_scraped)
d = page_info.word_dict
for j in range(len(d)):
word = d.keys()[j]
count = d.values()[j]
print '{} was found {} times'.format(word,count)
import re
cnn_url_regex = re.compile('(?<=[.]cnn)[.]com')
# (?<=[.]cnn)[.]com regular expression does the following:
# 1) match '.com' exactly
# 2) then looking backwards from where '.com' was found it attempts to find '.cnn'
w = WebCrawler(cnn_url_regex)
w.start_crawling(['http://www.cnn.com/2012/02/24/world/americas/haiti-pm-resigns/index.html?hpt=hp_t3'],1)
