From: Benjamin Mako Hill <mako@atdot.cc>
Date: Mon, 27 Apr 2015 22:58:47 +0000 (-0700)
Subject: add old version of script from CDSW (Fall 2014)
X-Git-Url: https://projects.mako.cc/source/harrypotter-wikipedia-cdsw/commitdiff_plain/99d8060001f49cb9a5bfb3834c7d6101500e9a86?ds=sidebyside

add old version of script from CDSW (Fall 2014)
---

99d8060001f49cb9a5bfb3834c7d6101500e9a86
diff --git a/build_hpwp_dataset.py b/build_hpwp_dataset.py
new file mode 100644
index 0000000..841a2b9
--- /dev/null
+++ b/build_hpwp_dataset.py
@@ -0,0 +1,104 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+import json
+import requests
+
+import codecs
+
+# get_article_revisions is a function that takes an article title in
+# wikipedia and return a list of all the revisions and meatadata for
+# that article
+def get_article_revisions(title):
+    revisions = []
+
+    # create a base url for the api and then a normal url which is initially just a copy of it
+    wp_api_base = "http://en.wikipedia.org/w/api.php/?action=query&titles=%(article_title)s&prop=revisions&rvprop=flags|timestamp|user|size|ids&rvlimit=500&format=json"
+    wp_api_base = wp_api_base % {'article_title': title }
+    wp_api_url = wp_api_base
+
+    # we'll repeat this forever (i.e., we'll only stop when we find the "break" command)
+    while True:
+        # the first line open the urls but also handles unicode urls
+        call = requests.get(wp_api_url)
+        api_answer = json.loads(call.content)
+
+        # get the list of pages from the json object
+        pages = api_answer["query"]["pages"]
+
+        # for every pages (there should always be only one) get the revisions
+        for page in pages.keys():
+            query_revisions = pages[page]["revisions"]
+
+            # for every revision, we do first do cleaning up
+            for rev in query_revisions:
+                # lets continue/skip if the user is hidden
+                if rev.has_key("userhidden"):
+                    continue
+                
+                # 1: add a title field for the article because we're going to mix them together
+                rev["title"] = title
+
+                # 2: lets "recode" anon so it's true or false instead of present/missing
+                if rev.has_key("anon"):
+                    rev["anon"] = True
+                else:
+                    rev["anon"] = False
+
+                # 3: letst recode "minor" in the same way
+                if rev.has_key("minor"):
+                    rev["minor"] = True
+                else:
+                    rev["minor"] = False
+
+                # we're going to change the timestamp to make it work a little better in excel and similar
+                rev["timestamp"] = rev["timestamp"].replace("T", " ")
+                rev["timestamp"] = rev["timestamp"].replace("Z", "")
+
+                # finally save the revisions we've seen to a varaible
+                revisions.append(rev)
+
+        # if there is a query-continue, it means there are more
+        if 'query-continue' in api_answer.keys():
+            # we will grab the rvcontinue token, insert it, and head back to the start of the loop
+            rvcontinue = api_answer["query-continue"]["revisions"]["rvcontinue"]
+            wp_api_url = wp_api_base + "&rvcontinue=%(continue_from)s" % {'continue_from' : rvcontinue}
+        else:
+            # no continue means we're done
+            break
+
+    # return all the revisions for this page
+    return(revisions)
+
+category = "Harry_Potter"
+
+# we'll use another api called catscan2 to grab a list of pages in
+# categories and subcategories. it works like all the other apis we've
+# studied!
+url_catscan = 'http://tools.wmflabs.org/catscan2/catscan2.php?depth=10&categories=%(category)s&doit=1&format=json'
+url_catscan = url_catscan % {'category' : category}
+call = requests.get(url_catscan)
+articles = json.loads(call.content)
+articles = articles["*"][0]["a"]["*"]
+
+# open a filie to write all the output
+output = codecs.open("hp_wiki.csv", "wb", "utf-8")
+output.write(",".join(["title", "user", "timestamp", "size", "anon", "minor", "revid"]) + "\n")
+
+# for every article
+for article in articles:
+
+    # first grab tht title
+    title = article["a"]["title"]
+
+    # get the list of revisions from our function and then interating through it printinig it out
+    revisions = get_article_revisions(title)
+    for rev in revisions:
+        output.write(",".join(['"' + rev["title"] + '"', '"' + rev["user"] + '"',
+                               rev["timestamp"], str(rev["size"]), str(rev["anon"]),
+                               str(rev["minor"]), str(rev["revid"])]) + "\n")
+
+# close the file, we're done here!
+output.close()
+    
+