From: Benjamin Mako Hill <mako@atdot.cc>
Date: Mon, 27 Apr 2015 23:35:09 +0000 (-0700)
Subject: created python 3 version for COM597G
X-Git-Url: https://projects.mako.cc/source/harrypotter-wikipedia-cdsw/commitdiff_plain/89f6c8568e611b9740bbd0afed5fbfa536bf4198?ds=inline;hp=99d8060001f49cb9a5bfb3834c7d6101500e9a86

created python 3 version for COM597G
---

diff --git a/build_hpwp_dataset.py b/build_hpwp_dataset.py
index 841a2b9..7cb5816 100644
--- a/build_hpwp_dataset.py
+++ b/build_hpwp_dataset.py
@@ -1,11 +1,8 @@
 #!/usr/bin/env python
 # coding=utf-8
 
-import json
 import requests
 
-import codecs
-
 # get_article_revisions is a function that takes an article title in
 # wikipedia and return a list of all the revisions and meatadata for
 # that article
@@ -13,15 +10,23 @@ def get_article_revisions(title):
     revisions = []
 
     # create a base url for the api and then a normal url which is initially just a copy of it
-    wp_api_base = "http://en.wikipedia.org/w/api.php/?action=query&titles=%(article_title)s&prop=revisions&rvprop=flags|timestamp|user|size|ids&rvlimit=500&format=json"
-    wp_api_base = wp_api_base % {'article_title': title }
-    wp_api_url = wp_api_base
-
-    # we'll repeat this forever (i.e., we'll only stop when we find the "break" command)
+    # http://en.wikipedia.org/w/api.php/?action=query&titles=%(article_title)s&prop=revisions&rvprop=flags|timestamp|user|size|ids&rvlimit=500&format=json
+    wp_api_url = "http://en.wikipedia.org/w/api.php/"
+
+    parameters = {'action' : 'query',
+                  'titles' : title,
+                  'prop' : 'revisions',
+                  'rvprop' : 'flags|timestamp|user|size|ids',
+                  'rvlimit' : 500,
+                  'format' : 'json',
+                  'continue' : '' }
+
+    # we'll repeat this forever (i.e., we'll only stop when we find
+    # the "break" command)
     while True:
         # the first line open the urls but also handles unicode urls
-        call = requests.get(wp_api_url)
-        api_answer = json.loads(call.content)
+        call = requests.get(wp_api_url, params=parameters)
+        api_answer = call.json()
 
         # get the list of pages from the json object
         pages = api_answer["query"]["pages"]
@@ -33,20 +38,20 @@ def get_article_revisions(title):
             # for every revision, we do first do cleaning up
             for rev in query_revisions:
                 # lets continue/skip if the user is hidden
-                if rev.has_key("userhidden"):
+                if "userhidden" in rev:
                     continue
                 
                 # 1: add a title field for the article because we're going to mix them together
                 rev["title"] = title
 
                 # 2: lets "recode" anon so it's true or false instead of present/missing
-                if rev.has_key("anon"):
+                if "anon" in rev:
                     rev["anon"] = True
                 else:
                     rev["anon"] = False
 
                 # 3: letst recode "minor" in the same way
-                if rev.has_key("minor"):
+                if "minor" in rev:
                     rev["minor"] = True
                 else:
                     rev["minor"] = False
@@ -58,31 +63,34 @@ def get_article_revisions(title):
                 # finally save the revisions we've seen to a varaible
                 revisions.append(rev)
 
-        # if there is a query-continue, it means there are more
-        if 'query-continue' in api_answer.keys():
-            # we will grab the rvcontinue token, insert it, and head back to the start of the loop
-            rvcontinue = api_answer["query-continue"]["revisions"]["rvcontinue"]
-            wp_api_url = wp_api_base + "&rvcontinue=%(continue_from)s" % {'continue_from' : rvcontinue}
+        if 'continue' in api_answer:
+            parameters.update(api_answer['continue'])
         else:
-            # no continue means we're done
             break
 
     # return all the revisions for this page
     return(revisions)
 
-category = "Harry_Potter"
+category = "Harry Potter"
 
 # we'll use another api called catscan2 to grab a list of pages in
 # categories and subcategories. it works like all the other apis we've
 # studied!
-url_catscan = 'http://tools.wmflabs.org/catscan2/catscan2.php?depth=10&categories=%(category)s&doit=1&format=json'
-url_catscan = url_catscan % {'category' : category}
-call = requests.get(url_catscan)
-articles = json.loads(call.content)
-articles = articles["*"][0]["a"]["*"]
+#
+# http://tools.wmflabs.org/catscan2/catscan2.php?depth=10&categories=%s&doit=1&format=json
+url_catscan = "http://tools.wmflabs.org/catscan2/catscan2.php"
+
+parameters = {'depth' : 10,
+              'categories' : category,
+              'format' : 'json',
+              'doit' : 1}
+
+r = requests.get(url_catscan, params=parameters)
+articles_json = r.json()
+articles = articles_json["*"][0]["a"]["*"]
 
 # open a filie to write all the output
-output = codecs.open("hp_wiki.csv", "wb", "utf-8")
+output = open("hp_wiki.csv", "w")
 output.write(",".join(["title", "user", "timestamp", "size", "anon", "minor", "revid"]) + "\n")
 
 # for every article