adding pullcat command
[mw] / src / mw / clicommands.py
1 ###
2 # mw - VCS-like nonsense for MediaWiki websites
3 # Copyright (C) 2010  Ian Weller <ian@ianweller.org>
4 #
5 # This program is free software; you can redistribute it and/or modify
6 # it under the terms of the GNU General Public License as published by
7 # the Free Software Foundation; either version 2 of the License, or
8 # (at your option) any later version.
9 #
10 # This program is distributed in the hope that it will be useful,
11 # but WITHOUT ANY WARRANTY; without even the implied warranty of
12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
13 # GNU General Public License for more details.
14 #
15 # You should have received a copy of the GNU General Public License along
16 # with this program.  If not, see <http://www.gnu.org/licenses/>.
17 ###
18
19 import codecs
20 import getpass
21 import hashlib
22 import mw.api
23 import mw.metadir
24 from optparse import OptionParser, OptionGroup
25 import os
26 import sys
27 import time
28
29
30 class CommandBase(object):
31
32     def __init__(self, name, description, usage=None):
33         self.me = os.path.basename(sys.argv[0])
34         self.description = description
35         if usage is None:
36             usage = '%prog ' + name
37         else:
38             usage = '%%prog %s %s' % (name, usage)
39         self.parser = OptionParser(usage=usage, description=description)
40         self.name = name
41         self.metadir = mw.metadir.Metadir()
42         self.shortcuts = []
43
44     def main(self):
45         (self.options, self.args) = self.parser.parse_args()
46         self.args = self.args[1:] # don't need the first thing
47         self._do_command()
48
49     def _do_command(self):
50         pass
51
52     def _login(self):
53         user = raw_input('Username: ')
54         passwd = getpass.getpass()
55         result = self.api.call({'action': 'login',
56                                 'lgname': user,
57                                 'lgpassword': passwd})
58         if result['login']['result'] == 'Success':
59             # cookies are saved to a file
60             print 'Login successful! (yay)'
61         elif result['login']['result'] == 'NeedToken':
62             print 'Login with token'
63             result = self.api.call({'action': 'login',
64                                     'lgname': user,
65                                     'lgpassword': passwd,
66                                     'lgtoken': result['login']['token']})
67             if result['login']['result'] == 'Success':
68                 print 'Login successful! (yay)'
69             else:
70                 print 'Login failed: %s' % result['login']['result']
71         else:
72             print 'Login failed: %s' % result['login']['result']
73
74     def _die_if_no_init(self):
75         if self.metadir.config is None:
76             print '%s: not a mw repo' % self.me
77             sys.exit(1)
78
79     def _api_setup(self):
80         self.api_url = self.metadir.config.get('remote', 'api_url')
81         self.api = mw.api.API(self.api_url, self.metadir)
82
83
84 class InitCommand(CommandBase):
85
86     def __init__(self):
87         usage = 'API_URL'
88         CommandBase.__init__(self, 'init', 'start a mw repo', usage)
89
90     def _do_command(self):
91         if len(self.args) < 1:
92             self.parser.error('must have URL to remote api.php')
93         elif len(self.args) > 1:
94             self.parser.error('too many arguments')
95         self.metadir.create(self.args[0])
96
97
98 class LoginCommand(CommandBase):
99
100     def __init__(self):
101         CommandBase.__init__(self, 'login', 'authenticate with wiki')
102
103     def _do_command(self):
104         self._die_if_no_init()
105         self._api_setup()
106         self._login()
107
108
109 class LogoutCommand(CommandBase):
110
111     def __init__(self):
112         CommandBase.__init__(self, 'logout', 'forget authentication')
113
114     def _do_command(self):
115         self._die_if_no_init()
116         try:
117             os.unlink(os.path.join(self.metadir.location, 'cookies'))
118         except OSError:
119             pass
120
121 class PullCategoryMembersCommand(CommandBase):
122
123     def __init__(self):
124         usage = '[options] PAGENAME ...'
125         CommandBase.__init__(self, 'pullcat', 'add remote pages to repo belonging to the given category', usage)
126
127     def _do_command(self):
128         self._die_if_no_init()
129         self._api_setup()
130         pages = []
131         pages += self.args
132         for these_pages in [pages[i:i + 25] for i in range(0, len(pages), 25)]:
133 #http://commons.wikimedia.org/w/api.php?action=query&format=xmlfm&generator=categorymembers&gcmlimit=500&gcmtitle=Category:User:Esby
134               data = {
135                       'action': 'query',
136                       'gcmtitle': '|'.join(these_pages),
137                       'generator' : 'categorymembers',
138                       'gcmlimit' : 500
139               }
140         response = self.api.call(data)['query']['pages']
141         for pageid in response.keys():
142           pagename = response[pageid]['title']
143           print pagename
144           pullc = PullCommand()
145           pullc.args = [pagename.encode('utf-8')]
146           pullc._do_command()
147
148
149 class PullCommand(CommandBase):
150
151     def __init__(self):
152         usage = '[options] PAGENAME ...'
153         CommandBase.__init__(self, 'pull', 'add remote pages to repo', usage)
154
155     def _do_command(self):
156         self._die_if_no_init()
157         self._api_setup()
158         pages = []
159         pages += self.args
160         for these_pages in [pages[i:i + 25] for i in range(0, len(pages), 25)]:
161             data = {
162                     'action': 'query',
163                     'titles': '|'.join(these_pages),
164                     'prop': 'info|revisions',
165                     'rvprop': 'ids|flags|timestamp|user|comment|content',
166             }
167             response = self.api.call(data)['query']['pages']
168             for pageid in response.keys():
169                 pagename = response[pageid]['title']
170                 if 'missing' in response[pageid].keys():
171                     print '%s: %s: page does not exist, file not created' % \
172                             (self.me, pagename)
173                     continue
174                 revids = [x['revid'] for x in response[pageid]['revisions']]
175                 revids.sort()
176                 self.metadir.pagedict_add(pagename, pageid, revids[-1])
177                 self.metadir.pages_add_rv(int(pageid),
178                                           response[pageid]['revisions'][0])
179                 filename = mw.api.pagename_to_filename(pagename)
180                 with file(os.path.join(self.metadir.root, filename + '.wiki'),
181                           'w') as fd:
182                     data = response[pageid]['revisions'][0]['*']
183                     data = data.encode('utf-8')
184                     fd.write(data)
185
186
187 class StatusCommand(CommandBase):
188
189     def __init__(self):
190         CommandBase.__init__(self, 'status', 'check repo status')
191         self.shortcuts.append('st')
192
193     def _do_command(self):
194         self._die_if_no_init()
195         status = self.metadir.working_dir_status()
196         for file in status:
197             print '%s %s' % (status[file], file)
198
199
200 class DiffCommand(CommandBase):
201
202     def __init__(self):
203         CommandBase.__init__(self, 'diff', 'diff wiki to working directory')
204
205     def _do_command(self):
206         self._die_if_no_init()
207         status = self.metadir.working_dir_status()
208         for file in status:
209             if status[file] == 'U':
210                 print self.metadir.diff_rv_to_working(
211                         mw.api.filename_to_pagename(file[:-5])),
212
213
214 class CommitCommand(CommandBase):
215
216     def __init__(self):
217         usage = '[FILES]'
218         CommandBase.__init__(self, 'commit', 'commit changes to wiki', usage)
219         self.shortcuts.append('ci')
220         self.parser.add_option('-m', '--message', dest='edit_summary',
221                                help='don\'t prompt for edit summary and '
222                                'use this instead')
223         self.parser.add_option('-b', '--bot', dest='bot', action='store_true',
224                                help='mark actions as a bot (won\'t affect '
225                                'anything if you don\'t have the bot right',
226                                default=False)
227
228     def _do_command(self):
229         self._die_if_no_init()
230         self._api_setup()
231         status = self.metadir.working_dir_status(files=self.args)
232         nothing_to_commit = True
233         for file in status:
234             print '%s %s' % (status[file], file)
235             if status[file] in ['U']:
236                 nothing_to_commit = False
237         if nothing_to_commit:
238             print 'nothing to commit'
239             sys.exit()
240         if self.options.edit_summary == None:
241             print 'Edit summary:',
242             edit_summary = raw_input()
243         else:
244             edit_summary = self.options.edit_summary
245         for file in status:
246             if status[file] in ['U']:
247                 # get edit token
248                 data = {
249                         'action': 'query',
250                         'prop': 'info|revisions',
251                         'intoken': 'edit',
252                         'titles': mw.api.filename_to_pagename(file[:-5]),
253                 }
254                 response = self.api.call(data)
255                 pageid = response['query']['pages'].keys()[0]
256                 revid = response['query']['pages'][pageid]['revisions'][0]\
257                         ['revid']
258                 awaitedrevid = self.metadir.pages_get_rv_list({'id': pageid})\
259                         [0]
260                 if revid != awaitedrevid:
261                     print 'warning: edit conflict detected on %s (%s -> %s) ' \
262                             '-- skipping!' % (file, awaitedrevid, revid)
263                     continue
264                 edittoken = response['query']['pages'][pageid]['edittoken']
265                 filename = os.path.join(self.metadir.root, file)
266                 text = codecs.open(filename, 'r', 'utf-8').read()
267                 text = text.encode('utf-8')
268                 if (len(text) != 0) and (text[-1] == '\n'):
269                     text = text[:-1]
270                 md5 = hashlib.md5()
271                 md5.update(text)
272                 textmd5 = md5.hexdigest()
273                 data = {
274                         'action': 'edit',
275                         'title': mw.api.filename_to_pagename(file[:-5]),
276                         'token': edittoken,
277                         'text': text,
278                         'md5': textmd5,
279                         'summary': edit_summary,
280                 }
281                 if self.options.bot:
282                     data['bot'] = 'bot'
283                 response = self.api.call(data)
284                 if response['edit']['result'] == 'Success':
285                     if 'nochange' in response['edit']:
286                         print 'warning: no changes detected in %s - ' \
287                                 'skipping and removing ending LF' % file
288                         self.metadir.clean_page(file[:-5])
289                         continue
290                     if response['edit']['oldrevid'] != revid:
291                         print 'warning: edit conflict detected on %s -- ' \
292                                 'skipping!' % file
293                         continue
294                     data = {
295                             'action': 'query',
296                             'revids': response['edit']['newrevid'],
297                             'prop': 'info|revisions',
298                             'rvprop':
299                                     'ids|flags|timestamp|user|comment|content',
300                     }
301                     response = self.api.call(data)['query']['pages']
302                     self.metadir.pages_add_rv(int(pageid),
303                                               response[pageid]['revisions'][0])
304                     print 'waiting 10s before processing the next file'
305                     time.sleep(10)
306                 else:
307                     print 'error: committing %s failed: %s' % \
308                             (file, response['edit']['result'])

Benjamin Mako Hill || Want to submit a patch?