X-Git-Url: https://projects.mako.cc/source/mw/blobdiff_plain/1d1aafa5f78d4db3df784026af179409e914e3d0..d18bf4d870cf3c3561aa4611345c824af77c1e32:/src/mw/clicommands.py diff --git a/src/mw/clicommands.py b/src/mw/clicommands.py index 7931ac6..3be6da3 100644 --- a/src/mw/clicommands.py +++ b/src/mw/clicommands.py @@ -1,6 +1,6 @@ ### # mw - VCS-like nonsense for MediaWiki websites -# Copyright (C) 2010 Ian Weller +# Copyright (C) 2011 Ian Weller and others # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -17,13 +17,15 @@ ### import codecs +import cookielib import getpass import hashlib -import mw.api import mw.metadir from optparse import OptionParser, OptionGroup import os +import simplemediawiki import sys +import time class CommandBase(object): @@ -42,7 +44,7 @@ class CommandBase(object): def main(self): (self.options, self.args) = self.parser.parse_args() - self.args = self.args[1:] # don't need the first thing + self.args = self.args[1:] # don't need the first thing self._do_command() def _do_command(self): @@ -76,8 +78,10 @@ class CommandBase(object): sys.exit(1) def _api_setup(self): + cookie_filename = os.path.join(self.metadir.location, 'cookies') self.api_url = self.metadir.config.get('remote', 'api_url') - self.api = mw.api.API(self.api_url, self.metadir) + self.api = simplemediawiki.MediaWiki(self.api_url, + cookie_file=cookie_filename) class InitCommand(CommandBase): @@ -118,11 +122,12 @@ class LogoutCommand(CommandBase): pass -class PullCommand(CommandBase): +class PullCategoryMembersCommand(CommandBase): def __init__(self): usage = '[options] PAGENAME ...' - CommandBase.__init__(self, 'pull', 'add remote pages to repo', usage) + CommandBase.__init__(self, 'pullcat', 'add remote pages to repo ' + 'belonging to the given category', usage) def _do_command(self): self._die_if_no_init() @@ -130,6 +135,46 @@ class PullCommand(CommandBase): pages = [] pages += self.args for these_pages in [pages[i:i + 25] for i in range(0, len(pages), 25)]: + data = { + 'action': 'query', + 'gcmtitle': '|'.join(these_pages), + 'generator': 'categorymembers', + 'gcmlimit': 500 + } + response = self.api.call(data)['query']['pages'] + for pageid in response.keys(): + pagename = response[pageid]['title'] + print pagename + pullc = PullCommand() + pullc.args = [pagename.encode('utf-8')] + pullc._do_command() + + +class PullCommand(CommandBase): + + def __init__(self): + usage = '[options] PAGENAME ...' + CommandBase.__init__(self, 'pull', 'add remote pages to repo', usage) + + def _do_command(self): + self._die_if_no_init() + self._api_setup() + pages = [] + pages += self.args + + # Pull should work with pagename, filename, or working directory + converted_pages = [] + if pages == []: + pages = self.metadir.working_dir_status().keys() + for pagename in pages: + if '.wiki' in pagename: + converted_pages.append( + mw.metadir.filename_to_pagename(pagename[:-5])) + else: + converted_pages.append(pagename) + pages = converted_pages + + for these_pages in [pages[i:i + 25] for i in range(0, len(pages), 25)]: # XXX ? data = { 'action': 'query', 'titles': '|'.join(these_pages), @@ -137,21 +182,44 @@ class PullCommand(CommandBase): 'rvprop': 'ids|flags|timestamp|user|comment|content', } response = self.api.call(data)['query']['pages'] + # for every pageid, returns dict.keys() = {'lastrevid', 'pageid', 'title', 'counter', 'length', 'touched': u'2011-02-02T19:32:04Z', 'ns', 'revisions' {...}} for pageid in response.keys(): pagename = response[pageid]['title'] + + # XXX is the revisions list a sorted one, should I use [0] or [-1]? + last_wiki_rev_comment = response[pageid]['revisions'][0]['comment'] + last_wiki_rev_user = response[pageid]['revisions'][0]['user'] + + # check if working file is modified or if wiki page doesn't exists + status = self.metadir.working_dir_status() + filename = mw.metadir.pagename_to_filename(pagename) + full_filename = os.path.join(self.metadir.root, filename + '.wiki') + if filename + '.wiki' in status and \ + status[filename + '.wiki' ] in ['M']: + print 'skipping: %s -- uncommitted modifications ' % (pagename) + continue if 'missing' in response[pageid].keys(): print '%s: %s: page does not exist, file not created' % \ (self.me, pagename) continue - revids = [x['revid'] for x in response[pageid]['revisions']] - revids.sort() - self.metadir.pagedict_add(pagename, pageid, revids[-1]) - self.metadir.pages_add_rv(int(pageid), - response[pageid]['revisions'][0]) - filename = mw.api.pagename_to_filename(pagename) - fd = file(os.path.join(self.metadir.root, filename + '.wiki'), - 'w') - fd.write(response[pageid]['revisions'][0]['*'].encode('utf-8')) + + wiki_revids = sorted([x['revid'] for x in response[pageid]['revisions']]) + last_wiki_revid = wiki_revids[-1] + working_revids = sorted(self.metadir.pages_get_rv_list({'id' : pageid})) + last_working_revid = working_revids[-1] + if ( os.path.exists(full_filename) and + last_wiki_revid == last_working_revid): + print 'wiki unchanged: %s' % (pagename) + else: + print 'pulling: %s : %s -- %s' % ( + pagename, last_wiki_rev_comment, last_wiki_rev_user) + self.metadir.pagedict_add(pagename, pageid, last_wiki_revid) + self.metadir.pages_add_rv(int(pageid), + response[pageid]['revisions'][0]) + with file(full_filename, 'w') as fd: + data = response[pageid]['revisions'][0]['*'] + data = data.encode('utf-8') + fd.write(data) class StatusCommand(CommandBase): @@ -163,8 +231,8 @@ class StatusCommand(CommandBase): def _do_command(self): self._die_if_no_init() status = self.metadir.working_dir_status() - for file in status: - print '%s %s' % (status[file], file) + for filename in status: + print '%s %s' % (status[filename], filename) class DiffCommand(CommandBase): @@ -175,10 +243,10 @@ class DiffCommand(CommandBase): def _do_command(self): self._die_if_no_init() status = self.metadir.working_dir_status() - for file in status: - if status[file] == 'U': + for filename in status: + if status[filename] == 'M': print self.metadir.diff_rv_to_working( - mw.api.filename_to_pagename(file[:-5])), + mw.metadir.filename_to_pagename(filename[:-5])), class CommitCommand(CommandBase): @@ -200,37 +268,40 @@ class CommitCommand(CommandBase): self._api_setup() status = self.metadir.working_dir_status(files=self.args) nothing_to_commit = True - for file in status: - print '%s %s' % (status[file], file) - if status[file] in ['U']: + for filename in status: + print '%s %s' % (status[filename], filename) + if status[filename] in ['M']: nothing_to_commit = False if nothing_to_commit: print 'nothing to commit' sys.exit() - print - print 'WARNING: mw does not do collision detection yet.' - print 'Hit ^C now if you haven\'t double checked, otherwise hit Enter' - raw_input() if self.options.edit_summary == None: print 'Edit summary:', edit_summary = raw_input() else: edit_summary = self.options.edit_summary - for file in status: - if status[file] in ['U']: + for file_num, filename in enumerate(status): + if status[filename] in ['M']: # get edit token data = { 'action': 'query', - 'prop': 'info', + 'prop': 'info|revisions', 'intoken': 'edit', - 'titles': mw.api.filename_to_pagename(file[:-5]), + 'titles': mw.metadir.filename_to_pagename(filename[:-5]), } response = self.api.call(data) - pageid = response['query']['pages'].keys()[0] - edittoken = response['query']['pages'][pageid]['edittoken'] - # FIXME use basetimestamp and starttimestamp - filename = os.path.join(self.metadir.root, file) - text = codecs.open(filename, 'r', 'utf-8').read() + pages = response['query']['pages'] + pageid = pages.keys()[0] + revid = pages[pageid]['revisions'][0]['revid'] + awaitedrevid = \ + self.metadir.pages_get_rv_list({'id': pageid})[0] + if revid != awaitedrevid: + print 'warning: edit conflict detected on %s (%s -> %s) ' \ + '-- skipping!' % (file, awaitedrevid, revid) + continue + edittoken = pages[pageid]['edittoken'] + full_filename = os.path.join(self.metadir.root, filename) + text = codecs.open(full_filename, 'r', 'utf-8').read() text = text.encode('utf-8') if (len(text) != 0) and (text[-1] == '\n'): text = text[:-1] @@ -239,7 +310,7 @@ class CommitCommand(CommandBase): textmd5 = md5.hexdigest() data = { 'action': 'edit', - 'title': mw.api.filename_to_pagename(file[:-5]), + 'title': mw.metadir.filename_to_pagename(filename[:-5]), 'token': edittoken, 'text': text, 'md5': textmd5, @@ -248,7 +319,23 @@ class CommitCommand(CommandBase): if self.options.bot: data['bot'] = 'bot' response = self.api.call(data) + if 'error' in response: + if 'code' in response['error']: + if response['error']['code'] == 'permissiondenied': + print 'Permission denied -- try running "mw login"' + return if response['edit']['result'] == 'Success': + if 'nochange' in response['edit']: + print 'warning: no changes detected in %s - ' \ + 'skipping and removing ending LF' % filename + pagename = mw.metadir.filename_to_pagename(filename[:-5]) + self.metadir.clean_page(pagename) + continue + if response['edit']['oldrevid'] != revid: + print 'warning: edit conflict detected on %s (%s -> %s) ' \ + '-- skipping!' % (file, + response['edit']['oldrevid'], revid) + continue data = { 'action': 'query', 'revids': response['edit']['newrevid'], @@ -259,6 +346,15 @@ class CommitCommand(CommandBase): response = self.api.call(data)['query']['pages'] self.metadir.pages_add_rv(int(pageid), response[pageid]['revisions'][0]) + # need to write latest rev to file too, as text may be changed + # such as a signature + with file(full_filename, 'w') as fd: + data = response[pageid]['revisions'][0]['*'] + data = data.encode('utf-8') + fd.write(data) + if file_num != len(status) - 1: + print 'waiting 3s before processing the next file' + time.sleep(3) else: - print 'committing %s failed: %s' % \ - (file, response['edit']['result']) + print 'error: committing %s failed: %s' % \ + (filename, response['edit']['result'])