From: Benjamin Mako Hill Date: Fri, 18 Jan 2019 06:22:06 +0000 (-0800) Subject: catscan is now petscan X-Git-Url: https://projects.mako.cc/source/harrypotter-wikipedia-cdsw/commitdiff_plain/ce5c13c094d659125fe85d59b9bc0e4c2bf40072?hp=2e473702cd932f0a9f9c05545134c372089ef30b catscan is now petscan - change category API endpoint - other minor changes to the API --- diff --git a/build_hpwp_dataset.py b/build_hpwp_dataset.py index 9d381ae..b421072 100644 --- a/build_hpwp_dataset.py +++ b/build_hpwp_dataset.py @@ -86,7 +86,7 @@ category = "Harry Potter" # # The following requests call basically does the same thing as this string: # "http://tools.wmflabs.org/catscan2/catscan2.php?depth=10&categories={0}&doit=1&format=json".format(category) -url_catscan = "http://tools.wmflabs.org/catscan3/catscan2.php" +url_catscan = "https://petscan.wmflabs.org/" parameters = {'depth' : 10, 'categories' : category, @@ -97,7 +97,7 @@ parameters = {'depth' : 10, r = requests.get(url_catscan, params=parameters) articles_json = r.json() -articles = articles_json["*"][0]["*"] +articles = articles_json["*"][0]["a"]["*"] # open a file to write all the output output = open("hp_wiki.tsv", "w", encoding="utf-8") @@ -105,9 +105,12 @@ output.write("\t".join(["title", "user", "timestamp", "size", "anon", "minor", " # for every article for article in articles: + # skip this until it's an article + if article["namespace"] != 0: + continue # first grab the article's title - title = article["a"]["title"] + title = article["title"] print(title) # get the list of revisions from our function and then iterate through it, @@ -120,5 +123,4 @@ for article in articles: # close the file, we're done here! output.close() - - +