X-Git-Url: https://projects.mako.cc/source/harrypotter-wikipedia-cdsw/blobdiff_plain/7a786062b74eb0335b21b1bd2ae860ea349748f8..3248892a26a9f80a1a8d6ef5da9ad89a26ca03df:/build_hpwp_dataset.py

diff --git a/build_hpwp_dataset.py b/build_hpwp_dataset.py
index 38c299d..b421072 100644
--- a/build_hpwp_dataset.py
+++ b/build_hpwp_dataset.py
@@ -1,6 +1,7 @@
 #!/usr/bin/env python
 # coding=utf-8
 
+import encoding_fix
 import requests
 
 # get_article_revisions is a function that takes an article title in
@@ -12,7 +13,7 @@ def get_article_revisions(title):
     # create a base url for the api and then a normal url which is initially
     # just a copy of it
     # The following line is what the requests call is doing, basically.
-    # "http://en.wikipedia.org/w/api.php/?action=query&titles={0}&prop=revisions&rvprop=flags|timestamp|user|size|ids&rvlimit=500&format=json".format(title)
+    # "http://en.wikipedia.org/w/api.php/?action=query&titles={0}&prop=revisions&rvprop=flags|timestamp|user|size|ids&rvlimit=500&format=json&continue=".format(title)
     wp_api_url = "http://en.wikipedia.org/w/api.php/"
 
     parameters = {'action' : 'query',
@@ -39,6 +40,7 @@ def get_article_revisions(title):
 
             # for every revision, first we do some cleaning up
             for rev in query_revisions:
+                #print(rev)
                 # let's continue/skip this revision if the user is hidden
                 if "userhidden" in rev:
                     continue
@@ -84,13 +86,15 @@ category = "Harry Potter"
 #
 # The following requests call basically does the same thing as this string:
 # "http://tools.wmflabs.org/catscan2/catscan2.php?depth=10&categories={0}&doit=1&format=json".format(category)
-url_catscan = "http://tools.wmflabs.org/catscan2/catscan2.php"
+url_catscan = "https://petscan.wmflabs.org/"
 
 parameters = {'depth' : 10,
               'categories' : category,
               'format' : 'json',
               'doit' : 1}
 
+# r = requests.get("http://tools.wmflabs.org/catscan2/catscan2.php?depth=10&categories=Harry Potter&doit=1&format=json"
+
 r = requests.get(url_catscan, params=parameters)
 articles_json = r.json()
 articles = articles_json["*"][0]["a"]["*"]
@@ -101,9 +105,13 @@ output.write("\t".join(["title", "user", "timestamp", "size", "anon", "minor", "
 
 # for every article
 for article in articles:
+    # skip this until it's an article
+    if article["namespace"] != 0:
+        continue
 
     # first grab the article's title
-    title = article["a"]["title"]
+    title = article["title"]
+    print(title)
 
     # get the list of revisions from our function and then iterate through it,
     # printing it to our output file
@@ -115,5 +123,4 @@ for article in articles:
 
 # close the file, we're done here!
 output.close()
-    
-    
+