forked from randomjohn/project
-
Notifications
You must be signed in to change notification settings - Fork 0
/
get_feed.py
90 lines (77 loc) · 2.84 KB
/
get_feed.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
# -*- coding: utf-8 -*-
# the purpose of this file is to extract the text contents from blogs that are given in a list
# (either feedlist.txt or one supplied as an argument)
import os
import sys
import re
from datetime import datetime as dt
import json
import feedparser
from BeautifulSoup import BeautifulStoneSoup
from nltk import clean_html
import codecs
import link_extractor as le
import filename_munger as fm
# Example feed:
# http://feeds.feedburner.com/oreilly/radar/atom
def cleanHtml(html):
"""
Clean up any html
"""
return BeautifulStoneSoup(clean_html(html),
convertEntities=BeautifulStoneSoup.HTML_ENTITIES).contents
def get_feed(blogurl,bloglist):
"""
Get a blog and write its contents out to a json file
"""
try:
feed_url = le.extract_feed_link_from_url(blogurl)
fp = feedparser.parse(feed_url)
except:
"Unable to retrieve or parse %s" % blogurl
return
try:
print >> sys.stderr, "Fetched %s entries from '%s'" % (len(fp.entries[0].title.encode('ascii','ignore')), fp.feed.title.encode('ascii','ignore'))
except IndexError:
print >> sys.stderr, "Retrieved no entries from '%s'" % feed_url
return None
blog_data = {'blogurl':blogurl,'title': fp.feed.title, 'blogroll':le.extract_links_from_url(blogurl,bloglist)}
blog_posts = [blog_data]
for e in fp.entries:
try:
blog_posts.append({'blogtitle':fp.feed.title,
'content': cleanHtml(e.content[0].value),
'link': e.links[0].href,
'links':le.extract_links(e.content[0].value),
'bloglinks':le.extract_links_from_list(e.content[0].value,bloglist)
})
except AttributeError:
blog_posts.append({'blogtitle':fp.feed.title,
'content': cleanHtml(e.summary),
'link': e.links[0].href,
'links':le.extract_links(e.summary),
'bloglinks':le.extract_links_from_list(e.summary,bloglist)
})
if not os.path.isdir('out'):
os.mkdir('out')
#out_file = '%s__%s.json' % (fp.feed.title.replace("'","").replace("-",""), dt.utcnow())
out_file = '%s.json' % (fm.munge(fp.feed.title))
#out_file = 'foo.json'
f = codecs.open(os.path.join(os.getcwd(), 'out', out_file), 'w',encoding='iso-8859-1')
f.write(json.dumps(blog_posts))
f.close()
print >> sys.stderr, 'Wrote output file to %s' % (f.name, )
return f.name
def main():
if len(sys.argv)>1:
blog_file = sys.argv[1]
else:
# use a default if none supplied
blog_file='feedlist.txt'
apcount={}
wordcounts={}
bloglist=[line for line in file(blog_file)]
for blogurl in bloglist:
get_feed(blogurl,bloglist)
if __name__=="__main__":
main()