X-Git-Url: https://projects.mako.cc/source/harrypotter-wikipedia-cdsw/blobdiff_plain/682b2b0eff68be39626aad640d55e7028a979a07..3248892a26a9f80a1a8d6ef5da9ad89a26ca03df:/build_hpwp_dataset.py

diff --git a/build_hpwp_dataset.py b/build_hpwp_dataset.py
index 97cc533..b421072 100644
--- a/build_hpwp_dataset.py
+++ b/build_hpwp_dataset.py
@@ -1,6 +1,7 @@
 #!/usr/bin/env python
 # coding=utf-8
 
+import encoding_fix
 import requests
 
 # get_article_revisions is a function that takes an article title in
@@ -85,7 +86,7 @@ category = "Harry Potter"
 #
 # The following requests call basically does the same thing as this string:
 # "http://tools.wmflabs.org/catscan2/catscan2.php?depth=10&categories={0}&doit=1&format=json".format(category)
-url_catscan = "http://tools.wmflabs.org/catscan2/catscan2.php"
+url_catscan = "https://petscan.wmflabs.org/"
 
 parameters = {'depth' : 10,
               'categories' : category,
@@ -104,9 +105,13 @@ output.write("\t".join(["title", "user", "timestamp", "size", "anon", "minor", "
 
 # for every article
 for article in articles:
+    # skip this until it's an article
+    if article["namespace"] != 0:
+        continue
 
     # first grab the article's title
-    title = article["a"]["title"]
+    title = article["title"]
+    print(title)
 
     # get the list of revisions from our function and then iterate through it,
     # printing it to our output file
@@ -118,5 +123,4 @@ for article in articles:
 
 # close the file, we're done here!
 output.close()
-    
-    
+