Update scan-feeds so it can find kcr's links.
[iron-blogger] / scan-feeds.py
1 #!/usr/bin/python
2 import yaml
3 import feedparser
4 import datetime
5 from dateutil.parser import parse
6 import dateutil.tz as tz
7
8 with open('bloggers.yml') as f:
9     users = yaml.safe_load(f.read())
10
11 log = {}
12
13 START = datetime.datetime(2009, 12, 21, 6)
14
15 def parse_published(pub):
16     return parse(pub).astimezone(tz.tzlocal()).replace(tzinfo=None)
17
18 def get_date(post):
19     if 'published' in post:
20         return post.published
21     return post.updated
22
23 def get_link(post):
24     if 'links' in post:
25         links = dict((l.rel, l) for l in post.links if 'html' in l.type)
26         if 'self' in links:
27             return links['self'].href
28         elif 'alternate' in links:
29             return links['alternate'].href
30     if 'href' in post:
31         return post.href
32     if 'link' in post:
33         return post.link
34     return None
35
36 def parse_feeds(weeks, uri):
37     feed = feedparser.parse(uri)
38     for post in feed.entries:
39         date = parse_published(get_date(post))
40
41         if date < START:
42             continue
43         wn = (date - START).days / 7
44
45         while len(weeks) <= wn:
46             weeks.append([])
47         weeks[wn].append(dict(
48                 date=date,
49                 title=post.title,
50                 url=get_link(post)))
51
52 for (username, u) in users.items():
53     weeks = []
54     print "[%s]" % (username)
55     for l in u['links']:
56         parse_feeds(weeks, l[2])
57     log[username] = weeks
58     for (i, w) in enumerate(weeks):
59         print " [%d]: %s" % (i, w)
60
61 with open('out/report.yml', 'w') as f:
62     yaml.safe_dump(log, f)

Benjamin Mako Hill || Want to submit a patch?