6 # get_article_revisions is a function that takes an article title in
7 # wikipedia and return a list of all the revisions and metadata for
9 def get_article_revisions(title):
12 # create a base url for the api and then a normal url which is initially
14 # The following line is what the requests call is doing, basically.
15 # "http://en.wikipedia.org/w/api.php/?action=query&titles={0}&prop=revisions&rvprop=flags|timestamp|user|size|ids&rvlimit=500&format=json&continue=".format(title)
16 wp_api_url = "http://en.wikipedia.org/w/api.php/"
18 parameters = {'action' : 'query',
21 'rvprop' : 'flags|timestamp|user|size|ids',
26 # we'll repeat this forever (i.e., we'll only stop when we find
27 # the "break" command)
29 # the first line open the urls but also handles unicode urls
30 call = requests.get(wp_api_url, params=parameters)
31 api_answer = call.json()
33 # get the list of pages from the json object
34 pages = api_answer["query"]["pages"]
36 # for every page, (there should always be only one) get its revisions:
37 for page in pages.keys():
38 query_revisions = pages[page]["revisions"]
40 # for every revision, first we do some cleaning up
41 for rev in query_revisions:
43 # let's continue/skip this revision if the user is hidden
44 if "userhidden" in rev:
47 # 1: add a title field for the article because we're going to mix them together
50 # 2: let's "recode" anon so it's true or false instead of present/missing
56 # 3: let's recode "minor" in the same way
62 # we're going to change the timestamp to make it work a little better in excel/spreadsheets
63 rev["timestamp"] = rev["timestamp"].replace("T", " ")
64 rev["timestamp"] = rev["timestamp"].replace("Z", "")
66 # finally, save the revisions we've seen to a varaible
69 # 'continue' tells us there's more revisions to add
70 if 'continue' in api_answer:
71 # replace the 'continue' parameter with the contents of the
72 # api_answer dictionary.
73 parameters.update(api_answer['continue'])
77 # return all the revisions for this page
80 category = "Harry Potter"
82 # we'll use another api called catscan2 to grab a list of pages in
83 # categories and subcategories. it works like all the other apis we've
86 # The following requests call basically does the same thing as this string:
87 # "http://tools.wmflabs.org/catscan2/catscan2.php?depth=10&categories={0}&doit=1&format=json".format(category)
88 url_catscan = "http://tools.wmflabs.org/catscan2/catscan2.php"
90 parameters = {'depth' : 10,
91 'categories' : category,
95 # r = requests.get("http://tools.wmflabs.org/catscan2/catscan2.php?depth=10&categories=Harry Potter&doit=1&format=json"
97 r = requests.get(url_catscan, params=parameters)
98 articles_json = r.json()
99 articles = articles_json["*"][0]["a"]["*"]
101 # open a file to write all the output
102 output = open("hp_wiki.tsv", "w", encoding="utf-8")
103 output.write("\t".join(["title", "user", "timestamp", "size", "anon", "minor", "revid"]) + "\n")
106 for article in articles:
108 # first grab the article's title
109 title = article["a"]["title"]
111 # get the list of revisions from our function and then iterate through it,
112 # printing it to our output file
113 revisions = get_article_revisions(title)
114 for rev in revisions:
115 output.write("\t".join(['"' + rev["title"] + '"', '"' + rev["user"] + '"',
116 rev["timestamp"], str(rev["size"]), str(rev["anon"]),
117 str(rev["minor"]), str(rev["revid"])]) + "\n")
119 # close the file, we're done here!