Merge git://github.com/esby/mw

[mw] / src / mw / clicommands.py

diff --git a/src/mw/clicommands.py b/src/mw/clicommands.py
index 088df16a5bf4c80330fd4d6956a152180af43901..9efffd4a2883a68476d1723befe4f446170b81b3 100644 (file)
--- a/src/mw/clicommands.py
+++ b/src/mw/clicommands.py
@@ -17,13 +17,15 @@
 ###
 
 import codecs
+import cookielib
 import getpass
 import hashlib
-import mw.api
 import mw.metadir
 from optparse import OptionParser, OptionGroup
 import os
+import simplemediawiki
 import sys
+import time
 
 
 class CommandBase(object):
@@ -76,8 +78,10 @@ class CommandBase(object):
             sys.exit(1)
 
     def _api_setup(self):
+        cookie_file = os.path.join(self.metadir.location, 'cookies')
         self.api_url = self.metadir.config.get('remote', 'api_url')
-        self.api = mw.api.API(self.api_url, self.metadir)
+        self.api = simplemediawiki.MediaWiki(self.api_url,
+                                             cookie_file=cookie_file)
 
 
 class InitCommand(CommandBase):
@@ -117,6 +121,33 @@ class LogoutCommand(CommandBase):
         except OSError:
             pass
 
+class PullCategoryMembersCommand(CommandBase):
+
+    def __init__(self):
+        usage = '[options] PAGENAME ...'
+        CommandBase.__init__(self, 'pullcat', 'add remote pages to repo belonging to the given category', usage)
+
+    def _do_command(self):
+        self._die_if_no_init()
+        self._api_setup()
+        pages = []
+        pages += self.args
+        for these_pages in [pages[i:i + 25] for i in range(0, len(pages), 25)]:
+#http://commons.wikimedia.org/w/api.php?action=query&format=xmlfm&generator=categorymembers&gcmlimit=500&gcmtitle=Category:User:Esby
+              data = {
+                      'action': 'query',
+                      'gcmtitle': '|'.join(these_pages),
+                      'generator' : 'categorymembers',
+                      'gcmlimit' : 500
+              }
+        response = self.api.call(data)['query']['pages']
+        for pageid in response.keys():
+          pagename = response[pageid]['title']
+          print pagename
+          pullc = PullCommand()
+          pullc.args = [pagename.encode('utf-8')]
+          pullc._do_command()
+
 
 class PullCommand(CommandBase):
 
@@ -148,12 +179,14 @@ class PullCommand(CommandBase):
                 self.metadir.pagedict_add(pagename, pageid, revids[-1])
                 self.metadir.pages_add_rv(int(pageid),
                                           response[pageid]['revisions'][0])
-                filename = mw.api.pagename_to_filename(pagename)
+                filename = mw.metadir.pagename_to_filename(pagename)
                 with file(os.path.join(self.metadir.root, filename + '.wiki'),
                           'w') as fd:
-                    data = response[pageid]['revisions'][0]['*'].encode('utf-8')
+                    data = response[pageid]['revisions'][0]['*']
+                    data = data.encode('utf-8')
                     fd.write(data)
 
+
 class StatusCommand(CommandBase):
 
     def __init__(self):
@@ -178,7 +211,7 @@ class DiffCommand(CommandBase):
         for file in status:
             if status[file] == 'U':
                 print self.metadir.diff_rv_to_working(
-                        mw.api.filename_to_pagename(file[:-5])),
+                        mw.metadir.filename_to_pagename(file[:-5])),
 
 
 class CommitCommand(CommandBase):
@@ -219,18 +252,19 @@ class CommitCommand(CommandBase):
                         'action': 'query',
                         'prop': 'info|revisions',
                         'intoken': 'edit',
-                        'titles': mw.api.filename_to_pagename(file[:-5]),
+                        'titles': mw.metadir.filename_to_pagename(file[:-5]),
                 }
                 response = self.api.call(data)
                 pageid = response['query']['pages'].keys()[0]
-                revid = response['query']['pages'][pageid]['revisions'][0]['revid']
-                awaitedrevid = self.metadir.pages_get_rv_list( {'id': pageid } )[0]                
+                revid = response['query']['pages'][pageid]['revisions'][0]\
+                        ['revid']
+                awaitedrevid = self.metadir.pages_get_rv_list({'id': pageid})\
+                        [0]
                 if revid != awaitedrevid:
-                    print "warning: edit conflict detected on %s (%s -> %s) " \
-                            "-- skipping!" % (file , awaitedrevid, revid)
+                    print 'warning: edit conflict detected on %s (%s -> %s) ' \
+                            '-- skipping!' % (file, awaitedrevid, revid)
                     continue
                 edittoken = response['query']['pages'][pageid]['edittoken']
-                # FIXME use basetimestamp and starttimestamp
                 filename = os.path.join(self.metadir.root, file)
                 text = codecs.open(filename, 'r', 'utf-8').read()
                 text = text.encode('utf-8')
@@ -241,7 +275,7 @@ class CommitCommand(CommandBase):
                 textmd5 = md5.hexdigest()
                 data = {
                         'action': 'edit',
-                        'title': mw.api.filename_to_pagename(file[:-5]),
+                        'title': mw.metadir.filename_to_pagename(file[:-5]),
                         'token': edittoken,
                         'text': text,
                         'md5': textmd5,
@@ -251,14 +285,14 @@ class CommitCommand(CommandBase):
                     data['bot'] = 'bot'
                 response = self.api.call(data)
                 if response['edit']['result'] == 'Success':
-                    if response['edit'].has_key('nochange'):
-                        print "warning: no changes detected in %s - " \
-                                "skipping and removing ending LF" % file
+                    if 'nochange' in response['edit']:
+                        print 'warning: no changes detected in %s - ' \
+                                'skipping and removing ending LF' % file
                         self.metadir.clean_page(file[:-5])
                         continue
                     if response['edit']['oldrevid'] != revid:
-                        print "warning: edit conflict detected on %s -- " \
-                                "skipping!" % file
+                        print 'warning: edit conflict detected on %s -- ' \
+                                'skipping!' % file
                         continue
                     data = {
                             'action': 'query',
@@ -270,6 +304,8 @@ class CommitCommand(CommandBase):
                     response = self.api.call(data)['query']['pages']
                     self.metadir.pages_add_rv(int(pageid),
                                               response[pageid]['revisions'][0])
+                    print 'waiting 10s before processing the next file'
+                    time.sleep(10)
                 else:
                     print 'error: committing %s failed: %s' % \
                             (file, response['edit']['result'])