From 79a83623784448efb08be8c68727ad1946ad648c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dan=20Michael=20O=2E=20Hegg=C3=B8?= <danmichaelo@gmail.com> Date: Sun, 16 Jun 2013 23:22:01 +0200 Subject: [PATCH] PEP8: Fix most remaining issues Fixed using: autopep8 mwclient -r -v -i -p 2000 --ignore=E24,W6,E501,W602 --- mwclient/__init__.py | 2 +- mwclient/client.py | 142 ++++++++++++++++++++---------------- mwclient/compatibility.py | 17 ++++- mwclient/errors.py | 21 ++++++ mwclient/ex.py | 34 ++++++--- mwclient/http.py | 72 ++++++++++++------ mwclient/listing.py | 45 ++++++++---- mwclient/page.py | 124 +++++++++++++++++++------------ mwclient/page_nowriteapi.py | 49 ++++++++----- mwclient/upload.py | 24 ++++-- 10 files changed, 342 insertions(+), 188 deletions(-) diff --git a/mwclient/__init__.py b/mwclient/__init__.py index 0289838..bb32dde 100644 --- a/mwclient/__init__.py +++ b/mwclient/__init__.py @@ -1,6 +1,6 @@ """ Copyright (c) 2006-2011 Bryan Tong Minh - + Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without diff --git a/mwclient/client.py b/mwclient/client.py index c1906af..eead6d6 100644 --- a/mwclient/client.py +++ b/mwclient/client.py @@ -1,8 +1,11 @@ __ver__ = '0.6.6' -import urllib, urlparse -import time, random -import sys, weakref +import urllib +import urlparse +import time +import random +import sys +import weakref import socket try: @@ -13,7 +16,8 @@ import http import upload import errors -import listing, page +import listing +import page import compatibility try: @@ -25,22 +29,28 @@ try: except ImportError: from StringIO import StringIO + def parse_timestamp(t): if t == '0000-00-00T00:00:00Z': return (0, 0, 0, 0, 0, 0, 0, 0) return time.strptime(t, '%Y-%m-%dT%H:%M:%SZ') + class WaitToken(object): + def __init__(self): self.id = '%x' % random.randint(0, sys.maxint) + def __hash__(self): return hash(self.id) + class Site(object): api_limit = 500 + def __init__(self, host, path='/w/', ext='.php', pool=None, retry_timeout=30, - max_retries=25, wait_callback=lambda *x: None, clients_useragent=None, - max_lag=3, compress=True, force_login=True, do_init=True): + max_retries=25, wait_callback=lambda *x: None, clients_useragent=None, + max_lag=3, compress=True, force_login=True, do_init=True): # Setup member variables self.host = host self.path = path @@ -59,7 +69,7 @@ class Site(object): # Site properties self.blocked = False # Whether current user is blocked - self.hasmsg = False # Whether current user has new messages + self.hasmsg = False # Whether current user has new messages self.groups = [] # Groups current user belongs to self.rights = [] # Rights current user has self.tokens = {} # Edit tokens of the current user @@ -95,10 +105,9 @@ class Site(object): if e[0] not in (u'unknown_action', u'readapidenied'): raise - def site_init(self): meta = self.api('query', meta='siteinfo|userinfo', - siprop='general|namespaces', uiprop='groups|rights') + siprop='general|namespaces', uiprop='groups|rights') # Extract site info self.site = meta['query']['general'] @@ -108,6 +117,7 @@ class Site(object): # Determine version if self.site['generator'].startswith('MediaWiki '): version = self.site['generator'][10:].split('.') + def split_num(s): i = 0 while i < len(s): @@ -134,15 +144,13 @@ class Site(object): self.rights = userinfo.get('rights', []) self.initialized = True - default_namespaces = {0: u'', 1: u'Talk', 2: u'User', 3: u'User talk', 4: u'Project', 5: u'Project talk', - 6: u'Image', 7: u'Image talk', 8: u'MediaWiki', 9: u'MediaWiki talk', 10: u'Template', 11: u'Template talk', - 12: u'Help', 13: u'Help talk', 14: u'Category', 15: u'Category talk', -1: u'Special', -2: u'Media'} + 6: u'Image', 7: u'Image talk', 8: u'MediaWiki', 9: u'MediaWiki talk', 10: u'Template', 11: u'Template talk', + 12: u'Help', 13: u'Help talk', 14: u'Category', 15: u'Category talk', -1: u'Special', -2: u'Media'} def __repr__(self): return "<Site object '%s%s'>" % (self.host, self.path) - def api(self, action, *args, **kwargs): """ An API call. Handles errors and returns dict object. """ kwargs.update(args) @@ -159,12 +167,12 @@ class Site(object): token = self.wait_token() while True: info = self.raw_api(action, **kwargs) - if not info: info = {} + if not info: + info = {} res = self.handle_api_result(info, token=token) if res: return info - def handle_api_result(self, info, kwargs=None, token=None): if token is None: token = self.wait_token() @@ -185,9 +193,9 @@ class Site(object): return False if '*' in info['error']: raise errors.APIError(info['error']['code'], - info['error']['info'], info['error']['*']) + info['error']['info'], info['error']['*']) raise errors.APIError(info['error']['code'], - info['error']['info'], kwargs) + info['error']['info'], kwargs) return True @staticmethod @@ -195,11 +203,12 @@ class Site(object): if type(data) is unicode: return data.encode('utf-8') return str(data) + @staticmethod def _query_string(*args, **kwargs): kwargs.update(args) qs = urllib.urlencode([(k, Site._to_str(v)) for k, v in kwargs.iteritems() - if k != 'wpEditToken']) + if k != 'wpEditToken']) if 'wpEditToken' in kwargs: qs += '&wpEditToken=' + urllib.quote(Site._to_str(kwargs['wpEditToken'])) return qs @@ -216,7 +225,7 @@ class Site(object): while True: try: stream = self.connection.post(self.host, - url, data=data, headers=headers) + url, data=data, headers=headers) if stream.getheader('Content-Encoding') == 'gzip': # BAD. seekable_stream = StringIO(stream.read()) @@ -270,13 +279,15 @@ class Site(object): self.wait_callback(self, token, retry, args) timeout = self.retry_timeout * retry - if timeout < min_wait: timeout = min_wait + if timeout < min_wait: + timeout = min_wait time.sleep(timeout) return self.wait_tokens[token] def require(self, major, minor, revision=None, raise_error=True): if self.version is None: - if raise_error is None: return + if raise_error is None: + return raise RuntimeError('Site %s has not yet been initialized' % repr(self)) if revision is None: @@ -284,21 +295,21 @@ class Site(object): return True elif raise_error: raise errors.MediaWikiVersionError('Requires version %s.%s, current version is %s.%s' - % ((major, minor) + self.version[:2])) + % ((major, minor) + self.version[:2])) else: return False else: raise NotImplementedError - # Actions def email(self, user, text, subject, cc=False): """Sends email to a specified user on the wiki.""" - #TODO: Use api! + # TODO: Use api! postdata = {} postdata['wpSubject'] = subject postdata['wpText'] = text - if cc: postdata['wpCCMe'] = '1' + if cc: + postdata['wpCCMe'] = '1' postdata['wpEditToken'] = self.tokens['edit'] postdata['uselang'] = 'en' postdata['title'] = u'Special:Emailuser/' + user @@ -310,10 +321,10 @@ class Site(object): raise errors.NoSpecifiedEmailError, user raise errors.EmailError, data - def login(self, username=None, password=None, cookies=None, domain=None): """Login to the wiki.""" - if self.initialized: self.require(1, 10) + if self.initialized: + self.require(1, 10) if username and password: self.credentials = (username, password, domain) @@ -327,7 +338,7 @@ class Site(object): kwargs = { 'lgname': self.credentials[0], 'lgpassword': self.credentials[1] - } + } if self.credentials[2]: kwargs['lgdomain'] = self.credentials[2] while True: @@ -351,21 +362,18 @@ class Site(object): else: self.site_init() - def upload(self, file=None, filename=None, description='', ignore=False, file_size=None, - url=None, session_key=None, comment=None): + url=None, session_key=None, comment=None): """Upload a file to the wiki.""" if self.version[:2] < (1, 16): return compatibility.old_upload(self, file=file, filename=filename, - description=description, ignore=ignore, - file_size=file_size) + description=description, ignore=ignore, + file_size=file_size) image = self.Images[filename] if not image.can('upload'): raise errors.InsufficientPermission(filename) - - predata = {} if comment is None: @@ -420,9 +428,12 @@ class Site(object): def parse(self, text=None, title=None, page=None): kwargs = {} - if text is not None: kwargs['text'] = text - if title is not None: kwargs['title'] = title - if page is not None: kwargs['page'] = page + if text is not None: + kwargs['text'] = text + if title is not None: + kwargs['title'] = title + if page is not None: + kwargs['page'] = page result = self.api('parse', **kwargs) return result['parse'] @@ -433,29 +444,30 @@ class Site(object): # Lists def allpages(self, start=None, prefix=None, namespace='0', filterredir='all', - minsize=None, maxsize=None, prtype=None, prlevel=None, - limit=None, dir='ascending', filterlanglinks='all', generator=True): + minsize=None, maxsize=None, prtype=None, prlevel=None, + limit=None, dir='ascending', filterlanglinks='all', generator=True): """Retrieve all pages on the wiki as a generator.""" self.require(1, 9) pfx = listing.List.get_prefix('ap', generator) kwargs = dict(listing.List.generate_kwargs(pfx, ('from', start), prefix=prefix, - minsize=minsize, maxsize=maxsize, prtype=prtype, prlevel=prlevel, - namespace=namespace, filterredir=filterredir, dir=dir, - filterlanglinks=filterlanglinks)) + minsize=minsize, maxsize=maxsize, prtype=prtype, prlevel=prlevel, + namespace=namespace, filterredir=filterredir, dir=dir, + filterlanglinks=filterlanglinks)) return listing.List.get_list(generator)(self, 'allpages', 'ap', limit=limit, return_values='title', **kwargs) # def allimages(self): requires 1.12 # TODO! def alllinks(self, start=None, prefix=None, unique=False, prop='title', - namespace='0', limit=None, generator=True): + namespace='0', limit=None, generator=True): """Retrieve a list of all links on the wiki as a generator.""" self.require(1, 11) pfx = listing.List.get_prefix('al', generator) kwargs = dict(listing.List.generate_kwargs(pfx, ('from', start), prefix=prefix, - prop=prop, namespace=namespace)) - if unique: kwargs[pfx + 'unique'] = '1' + prop=prop, namespace=namespace)) + if unique: + kwargs[pfx + 'unique'] = '1' return listing.List.get_list(generator)(self, 'alllinks', 'al', limit=limit, return_values='title', **kwargs) def allcategories(self, start=None, prefix=None, dir='ascending', limit=None, generator=True): @@ -471,11 +483,11 @@ class Site(object): self.require(1, 11) kwargs = dict(listing.List.generate_kwargs('au', ('from', start), prefix=prefix, - group=group, prop=prop)) + group=group, prop=prop)) return listing.List(self, 'allusers', 'au', limit=limit, **kwargs) def blocks(self, start=None, end=None, dir='older', ids=None, users=None, limit=None, - prop='id|user|by|timestamp|expiry|reason|flags'): + prop='id|user|by|timestamp|expiry|reason|flags'): """Retrieve blocks as a generator. Each block is a dictionary containing: @@ -493,16 +505,16 @@ class Site(object): self.require(1, 12) # TODO: Fix. Fix what? kwargs = dict(listing.List.generate_kwargs('bk', start=start, end=end, dir=dir, - users=users, prop=prop)) + users=users, prop=prop)) return listing.List(self, 'blocks', 'bk', limit=limit, **kwargs) def deletedrevisions(self, start=None, end=None, dir='older', namespace=None, - limit=None, prop='user|comment'): + limit=None, prop='user|comment'): # TODO: Fix self.require(1, 12) kwargs = dict(listing.List.generate_kwargs('dr', start=start, end=end, dir=dir, - namespace=namespace, prop=prop)) + namespace=namespace, prop=prop)) return listing.List(self, 'deletedrevs', 'dr', limit=limit, **kwargs) def exturlusage(self, query, prop=None, protocol='http', namespace=None, limit=None): @@ -526,15 +538,15 @@ class Site(object): self.require(1, 11) kwargs = dict(listing.List.generate_kwargs('eu', query=query, prop=prop, - protocol=protocol, namespace=namespace)) + protocol=protocol, namespace=namespace)) return listing.List(self, 'exturlusage', 'eu', limit=limit, **kwargs) def logevents(self, type=None, prop=None, start=None, end=None, - dir='older', user=None, title=None, limit=None, action=None): + dir='older', user=None, title=None, limit=None, action=None): self.require(1, 10) kwargs = dict(listing.List.generate_kwargs('le', prop=prop, type=type, start=start, - end=end, dir=dir, user=user, title=title, action=action)) + end=end, dir=dir, user=user, title=title, action=action)) return listing.List(self, 'logevents', 'le', limit=limit, **kwargs) # def protectedtitles requires 1.15 @@ -553,26 +565,27 @@ class Site(object): return listing.List(self, 'random', 'rn', limit=limit, **kwargs) def recentchanges(self, start=None, end=None, dir='older', namespace=None, - prop=None, show=None, limit=None, type=None): + prop=None, show=None, limit=None, type=None): self.require(1, 9) kwargs = dict(listing.List.generate_kwargs('rc', start=start, end=end, dir=dir, - namespace=namespace, prop=prop, show=show, type=type)) + namespace=namespace, prop=prop, show=show, type=type)) return listing.List(self, 'recentchanges', 'rc', limit=limit, **kwargs) def search(self, search, namespace='0', what='title', redirects=False, limit=None): self.require(1, 11) kwargs = dict(listing.List.generate_kwargs('sr', search=search, namespace=namespace, what=what)) - if redirects: kwargs['srredirects'] = '1' + if redirects: + kwargs['srredirects'] = '1' return listing.List(self, 'search', 'sr', limit=limit, **kwargs) def usercontributions(self, user, start=None, end=None, dir='older', namespace=None, - prop=None, show=None, limit=None): + prop=None, show=None, limit=None): self.require(1, 9) kwargs = dict(listing.List.generate_kwargs('uc', user=user, start=start, end=end, - dir=dir, namespace=namespace, prop=prop, show=show)) + dir=dir, namespace=namespace, prop=prop, show=show)) return listing.List(self, 'usercontribs', 'uc', limit=limit, **kwargs) def users(self, users, prop='blockinfo|groups|editcount'): @@ -581,12 +594,13 @@ class Site(object): return listing.List(self, 'users', 'us', ususers='|'.join(users), usprop=prop) def watchlist(self, allrev=False, start=None, end=None, namespace=None, dir='older', - prop=None, show=None, limit=None): + prop=None, show=None, limit=None): self.require(1, 9) kwargs = dict(listing.List.generate_kwargs('wl', start=start, end=end, - namespace=namespace, dir=dir, prop=prop, show=show)) - if allrev: kwargs['wlallrev'] = '1' + namespace=namespace, dir=dir, prop=prop, show=show)) + if allrev: + kwargs['wlallrev'] = '1' return listing.List(self, 'watchlist', 'wl', limit=limit, **kwargs) def expandtemplates(self, text, title=None, generatexml=False): @@ -594,8 +608,10 @@ class Site(object): self.require(1, 11) kwargs = {} - if title is None: kwargs['title'] = title - if generatexml: kwargs['generatexml'] = '1' + if title is None: + kwargs['title'] = title + if generatexml: + kwargs['generatexml'] = '1' result = self.api('expandtemplates', text=text, **kwargs) diff --git a/mwclient/compatibility.py b/mwclient/compatibility.py index f2d8794..273a16f 100644 --- a/mwclient/compatibility.py +++ b/mwclient/compatibility.py @@ -1,4 +1,6 @@ -import upload, errors +import upload +import errors + def title(prefix, new_format): if new_format: @@ -6,6 +8,7 @@ def title(prefix, new_format): else: return 'titles' + def userinfo(data, new_format=None): if new_format is None: # Unknown version; trying to guess @@ -20,6 +23,7 @@ def userinfo(data, new_format=None): else: return data['userinfo'] + def iiprop(version): if version[:2] >= (1, 13): return 'timestamp|user|comment|url|size|sha1|metadata|archivename' @@ -28,12 +32,14 @@ def iiprop(version): else: return 'timestamp|user|comment|url|size|sha1' + def cmtitle(page, new_format, prefix=''): if new_format: return prefix + 'title', page.name else: return prefix + 'category', page.strip_namespace(page.name) + def protectright(version): if version[:2] >= (1, 13): return 'editprotected' @@ -41,6 +47,8 @@ def protectright(version): return 'protect' from cStringIO import StringIO + + def old_upload(self, file, filename, description, license='', ignore=False, file_size=None): image = self.Images[filename] if not image.can('upload'): @@ -61,7 +69,8 @@ def old_upload(self, file, filename, description, license='', ignore=False, file # predata['wpDestFile'] = filename predata['wpUploadDescription'] = description predata['wpLicense'] = license - if ignore: predata['wpIgnoreWarning'] = 'true' + if ignore: + predata['wpIgnoreWarning'] = 'true' predata['wpUpload'] = 'Upload file' predata['wpSourceType'] = 'file' predata['wpDestFile'] = filename @@ -73,8 +82,8 @@ def old_upload(self, file, filename, description, license='', ignore=False, file while True: try: self.connection.post(self.host, - self.path + 'index.php?title=Special:Upload&maxlag=' - + self.max_lag, data=postdata).read() + self.path + 'index.php?title=Special:Upload&maxlag=' + + self.max_lag, data=postdata).read() except errors.HTTPStatusError, e: if e[0] == 503 and e[1].getheader('X-Database-Lag'): self.wait(wait_token, int(e[1].getheader('Retry-After'))) diff --git a/mwclient/errors.py b/mwclient/errors.py index 0a17132..4075013 100644 --- a/mwclient/errors.py +++ b/mwclient/errors.py @@ -1,37 +1,55 @@ class MwClientError(RuntimeError): pass + class MediaWikiVersionError(MwClientError): pass + class APIDisabledError(MwClientError): pass + class HTTPError(MwClientError): pass + + class HTTPStatusError(MwClientError): pass + + class HTTPRedirectError(HTTPError): pass + class MaximumRetriesExceeded(MwClientError): pass + class APIError(MwClientError): + def __init__(self, code, info, kwargs): self.code = code self.info = info MwClientError.__init__(self, code, info, kwargs) + class InsufficientPermission(MwClientError): pass + + class UserBlocked(InsufficientPermission): pass + class EditError(MwClientError): pass + + class ProtectedPageError(EditError, InsufficientPermission): pass + + class FileExists(EditError): pass @@ -39,7 +57,10 @@ class FileExists(EditError): class LoginError(MwClientError): pass + class EmailError(MwClientError): pass + + class NoSpecifiedEmail(EmailError): pass diff --git a/mwclient/ex.py b/mwclient/ex.py index 2e0f239..d0ec8da 100644 --- a/mwclient/ex.py +++ b/mwclient/ex.py @@ -1,4 +1,6 @@ -import client, http +import client +import http + def read_config(config_files, **predata): cfg = {} @@ -7,6 +9,7 @@ def read_config(config_files, **predata): config_file, predata)) return cfg + def _read_config_file(_config_file, predata): _file = open(_config_file) exec _file in globals(), predata @@ -19,17 +22,23 @@ def _read_config_file(_config_file, predata): if not _k.startswith('_'): yield _k, _v + class SiteList(object): + def __init__(self): self.sites = {} + def __getitem__(self, key): if key not in self.sites: self.sites[key] = {} return self.sites[key] + def __iter__(self): return self.sites.itervalues() + class ConfiguredSite(client.Site): + def __init__(self, *config_files, **kwargs): self.config = read_config(config_files, sites=SiteList()) @@ -39,23 +48,24 @@ class ConfiguredSite(client.Site): do_login = 'username' in self.config and 'password' in self.config client.Site.__init__(self, host=self.config['host'], - path=self.config['path'], ext=self.config.get('ext', '.php'), - do_init=not do_login, - retry_timeout=self.config.get('retry_timeout', 30), - max_retries=self.config.get('max_retries', -1)) - + path=self.config['path'], ext=self.config.get('ext', '.php'), + do_init=not do_login, + retry_timeout=self.config.get('retry_timeout', 30), + max_retries=self.config.get('max_retries', -1)) if do_login: self.login(self.config['username'], - self.config['password']) + self.config['password']) + class ConfiguredPool(list): + def __init__(self, *config_files): self.config = read_config(config_files, sites=SiteList()) self.pool = http.HTTPPool() config = dict([(k, v) for k, v in self.config.iteritems() - if k != 'sites']) + if k != 'sites']) for site in self.config['sites']: cfg = config.copy() @@ -65,10 +75,10 @@ class ConfiguredPool(list): do_login = 'username' in site and 'password' in site self.append(client.Site(host=site['host'], - path=site['path'], ext=site.get('ext', '.php'), - pool=self.pool, do_init=not do_login, - retry_timeout=site.get('retry_timeout', 30), - max_retries=site.get('max_retries', -1))) + path=site['path'], ext=site.get('ext', '.php'), + pool=self.pool, do_init=not do_login, + retry_timeout=site.get('retry_timeout', 30), + max_retries=site.get('max_retries', -1))) if do_login: self[-1].login(site['username'], site['password']) self[-1].config = site diff --git a/mwclient/http.py b/mwclient/http.py index 7fb3d54..b929850 100644 --- a/mwclient/http.py +++ b/mwclient/http.py @@ -9,17 +9,22 @@ import errors from client import __ver__ + class CookieJar(dict): + def __init__(self): dict.__init__(self, ()) + def extract_cookies(self, response): for cookie in response.msg.getallmatchingheaders('Set-Cookie'): self.parse_cookie(cookie.strip()) if response.getheader('set-cookie2', None): # ... raise RuntimeError, 'Set-Cookie2', value + def parse_cookie(self, cookie): - if not cookie: return + if not cookie: + return value, attrs = cookie.split(': ', 1)[1].split(';', 1) i = value.strip().split('=') if len(i) == 1 and i[0] in self: @@ -29,15 +34,19 @@ class CookieJar(dict): def get_cookie_header(self): return '; '.join(('%s=%s' % i for i in self.iteritems())) + def __iter__(self): for k, v in self.iteritems(): yield Cookie(k, v) + class Cookie(object): + def __init__(self, name, value): self.name = name self.value = value + class HTTPPersistentConnection(object): http_class = httplib.HTTPConnection scheme_name = 'http' @@ -50,14 +59,16 @@ class HTTPPersistentConnection(object): self.cookies = {} self.pool = pool - if pool: self.cookies = pool.cookies + if pool: + self.cookies = pool.cookies clients_useragent = clients_useragent or "" - if clients_useragent != "": clients_useragent += " " + if clients_useragent != "": + clients_useragent += " " self.useragent = clients_useragent + 'MwClient/' + __ver__ def request(self, method, host, path, headers, data, - raise_on_not_ok=True, auto_redirect=True): + raise_on_not_ok=True, auto_redirect=True): # Strip scheme if type(host) is tuple: @@ -82,7 +93,8 @@ class HTTPPersistentConnection(object): elif data: headers['Content-Length'] = str(len(data)) - if _headers: headers.update(_headers) + if _headers: + headers.update(_headers) try: self._conn.request(method, path, headers=headers) @@ -103,10 +115,11 @@ class HTTPPersistentConnection(object): except socket.error, e: self._conn.close() raise errors.HTTPError, e - #except Exception, e: + # except Exception, e: # raise errors.HTTPError, e - if not host in self.cookies: self.cookies[host] = CookieJar() + if not host in self.cookies: + self.cookies[host] = CookieJar() self.cookies[host].extract_cookies(res) if res.status >= 300 and res.status <= 399 and auto_redirect: @@ -122,16 +135,17 @@ class HTTPPersistentConnection(object): data = '' old_path = path path = location[2] - if location[4]: path = path + '?' + location[4] + if location[4]: + path = path + '?' + location[4] if location[0].lower() != self.scheme_name: raise errors.HTTPRedirectError, ('Only HTTP connections are supported', - res.getheader('Location')) + res.getheader('Location')) if self.pool is None: if location[1] != host: raise errors.HTTPRedirectError, ('Redirecting to different hosts not supported', - res.getheader('Location')) + res.getheader('Location')) return self.request(method, host, path, headers, data) else: @@ -139,7 +153,7 @@ class HTTPPersistentConnection(object): conn = self.__class__(location[1], self.pool) self.pool.append(([location[1]], conn)) return self.pool.request(method, location[1], path, - headers, data, raise_on_not_ok, auto_redirect) + headers, data, raise_on_not_ok, auto_redirect) if res.status != 200 and raise_on_not_ok: try: @@ -151,35 +165,43 @@ class HTTPPersistentConnection(object): def get(self, host, path, headers=None): return self.request('GET', host, path, headers, None) + def post(self, host, path, headers=None, data=None): return self.request('POST', host, path, headers, data) + def head(self, host, path, headers=None, auto_redirect=False): res = self.request('HEAD', host, path, headers, - data=None, raise_on_not_ok=False, - auto_redirect=auto_redirect) + data=None, raise_on_not_ok=False, + auto_redirect=auto_redirect) res.read() return res.status, res.getheaders() def close(self): self._conn.close() + def fileno(self): return self._conn.sock.fileno() + class HTTPConnection(HTTPPersistentConnection): + def request(self, method, host, path, headers, data, - raise_on_not_ok=True, auto_redirect=True): - if not headers: headers = {} + raise_on_not_ok=True, auto_redirect=True): + if not headers: + headers = {} headers['Connection'] = 'Close' res = HTTPPersistentConnection.request(self, method, host, path, headers, data, - raise_on_not_ok, auto_redirect) + raise_on_not_ok, auto_redirect) return res + class HTTPSPersistentConnection(HTTPPersistentConnection): http_class = httplib.HTTPSConnection scheme_name = 'https' class HTTPPool(list): + def __init__(self, clients_useragent=None): list.__init__(self) self.cookies = {} @@ -190,7 +212,8 @@ class HTTPPool(list): scheme, host = host for hosts, conn in self: - if (scheme, host) in hosts: return conn + if (scheme, host) in hosts: + return conn redirected_host = None for hosts, conn in self: @@ -214,19 +237,24 @@ class HTTPPool(list): conn = cls(host, self, self.clients_useragent) self.append(([(scheme, host)], conn)) return conn + def get(self, host, path, headers=None): return self.find_connection(host).get(host, - path, headers) + path, headers) + def post(self, host, path, headers=None, data=None): return self.find_connection(host).post(host, - path, headers, data) + path, headers, data) + def head(self, host, path, headers=None, auto_redirect=False): return self.find_connection(host).head(host, - path, headers, auto_redirect) + path, headers, auto_redirect) + def request(self, method, host, path, headers, data, - raise_on_not_ok, auto_redirect): + raise_on_not_ok, auto_redirect): return self.find_connection(host).request(method, host, path, - headers, data, raise_on_not_ok, auto_redirect) + headers, data, raise_on_not_ok, auto_redirect) + def close(self): for hosts, conn in self: conn.close() diff --git a/mwclient/listing.py b/mwclient/listing.py index 152c89d..0f5a4bb 100644 --- a/mwclient/listing.py +++ b/mwclient/listing.py @@ -1,7 +1,10 @@ -import client, page +import client +import page import compatibility + class List(object): + def __init__(self, site, list_name, prefix, limit=None, return_values=None, max_items=None, *args, **kwargs): # NOTE: Fix limit self.site = site @@ -12,7 +15,8 @@ class List(object): kwargs.update(args) self.args = kwargs - if limit is None: limit = site.api_limit + if limit is None: + limit = site.api_limit self.args[self.prefix + 'limit'] = str(limit) self.count = 0 @@ -36,7 +40,8 @@ class List(object): self.count += 1 if 'timestamp' in item: item['timestamp'] = client.parse_timestamp(item['timestamp']) - if full: return item + if full: + return item if type(self.return_values) is tuple: return tuple((item[i] for i in self.return_values)) @@ -46,7 +51,8 @@ class List(object): return item[self.return_values] except StopIteration: - if self.last: raise StopIteration + if self.last: + raise StopIteration self.load_chunk() return List.next(self, full=full) @@ -70,7 +76,6 @@ class List(object): else: self._iter = data['query'][self.result_member].itervalues() - def __repr__(self): return "<List object '%s' for %s>" % (self.list_name, self.site) @@ -97,6 +102,7 @@ class List(object): class GeneratorList(List): + def __init__(self, site, list_name, prefix, *args, **kwargs): List.__init__(self, site, list_name, prefix, *args, **kwargs) @@ -127,35 +133,41 @@ class GeneratorList(List): class Category(page.Page, GeneratorList): + def __init__(self, site, name, info=None, namespace=None): page.Page.__init__(self, site, name, info) kwargs = {} kwargs.update((compatibility.cmtitle(self, self.site.require( 1, 12, raise_error=False), prefix='gcm'), )) - if namespace: kwargs['gcmnamespace'] = namespace + if namespace: + kwargs['gcmnamespace'] = namespace GeneratorList.__init__(self, site, 'categorymembers', 'cm', **kwargs) def __repr__(self): return "<Category object '%s' for %s>" % (self.name.encode('utf-8'), self.site) def members(self, prop='ids|title', namespace=None, sort='sortkey', - dir='asc', start=None, end=None, generator=True): + dir='asc', start=None, end=None, generator=True): prefix = self.get_prefix('cm', generator) kwargs = dict(self.generate_kwargs(prefix, prop=prop, namespace=namespace, - sort=sort, dir=dir, start=start, end=end, *(compatibility.cmtitle( - self, self.site.require(1, 12, raise_error=False)), ))) + sort=sort, dir=dir, start=start, end=end, *(compatibility.cmtitle( + self, self.site.require(1, 12, raise_error=False)), ))) return self.get_list(generator)(self.site, 'categorymembers', 'cm', **kwargs) + class PageList(GeneratorList): + def __init__(self, site, prefix=None, start=None, namespace=0, redirects='all'): self.namespace = namespace kwargs = {} - if prefix: kwargs['apprefix'] = prefix - if start: kwargs['apfrom'] = start + if prefix: + kwargs['apprefix'] = prefix + if start: + kwargs['apfrom'] = start GeneratorList.__init__(self, site, 'allpages', 'ap', - apnamespace=str(namespace), apfilterredir=redirects, **kwargs) + apnamespace=str(namespace), apfilterredir=redirects, **kwargs) def __getitem__(self, name): return self.get(name, None) @@ -180,16 +192,18 @@ class PageList(GeneratorList): def guess_namespace(self, name): normal_name = page.Page.normalize_title(name) for ns in self.site.namespaces: - if ns == 0: continue + if ns == 0: + continue if name.startswith(u'%s:' % self.site.namespaces[ns].replace(' ', '_')): return ns elif ns in self.site.default_namespaces: - if name.startswith(u'%s:' % self.site.default_namespaces[ns].replace(' ', '_')): + if name.startswith(u'%s:' % self.site.default_namespaces[ns].replace(' ', '_')): return ns return 0 class PageProperty(List): + def __init__(self, page, prop, prefix, *args, **kwargs): List.__init__(self, page.site, prop, prefix, titles=page.name, *args, **kwargs) self.page = page @@ -204,11 +218,14 @@ class PageProperty(List): class PagePropertyGenerator(GeneratorList): + def __init__(self, page, prop, prefix, *args, **kwargs): GeneratorList.__init__(self, page.site, prop, prefix, titles=page.name, *args, **kwargs) self.page = page + class RevisionsIterator(PageProperty): + def load_chunk(self): if 'rvstartid' in self.args and 'rvstart' in self.args: del self.args['rvstart'] diff --git a/mwclient/page.py b/mwclient/page.py index 7ec1773..66e77da 100644 --- a/mwclient/page.py +++ b/mwclient/page.py @@ -1,10 +1,16 @@ -import client, errors, listing +import client +import errors +import listing import compatibility from page_nowriteapi import OldPage -import urllib, urlparse, time +import urllib +import urlparse +import time + class Page(object): + def __init__(self, site, name, info=None, extra_properties={}): if type(name) is type(self): return self.__dict__.update(name.__dict__) @@ -23,10 +29,10 @@ class Page(object): if type(name) is int: info = self.site.api('query', prop=prop, pageids=name, - inprop='protection', *extra_props) + inprop='protection', *extra_props) else: info = self.site.api('query', prop=prop, titles=name, - inprop='protection', *extra_props) + inprop='protection', *extra_props) info = info['query']['pages'].itervalues().next() self._info = info @@ -88,10 +94,10 @@ class Page(object): title = title.replace(' ', '_') return title - def can(self, action): level = self.protection.get(action, (action, ))[0] - if level == 'sysop': level = compatibility.protectright(self.site.version) + if level == 'sysop': + level = compatibility.protectright(self.site.version) return level in self.site.rights @@ -102,7 +108,7 @@ class Page(object): self.site.tokens[type] = '0' if self.site.tokens.get(type, '0') == '0' or force: info = self.site.api('query', titles=self.name, - prop='info', intoken=type) + prop='info', intoken=type) for i in info['query']['pages'].itervalues(): if i['title'] == self.name: self.site.tokens[type] = i['%stoken' % type] @@ -151,26 +157,34 @@ class Page(object): if not self.can('edit'): raise errors.ProtectedPageError(self) - if not text: text = self.text - if not section: section = self.section + if not text: + text = self.text + if not section: + section = self.section if not self.site.writeapi: return OldPage.save(self, text=text, summary=summary, minor=False) data = {} - if minor: data['minor'] = '1' - if not minor: data['notminor'] = '1' - if self.last_rev_time: data['basetimestamp'] = time.strftime('%Y%m%d%H%M%S', self.last_rev_time) - if self.edit_time: data['starttimestamp'] = time.strftime('%Y%m%d%H%M%S', self.edit_time) - if bot: data['bot'] = '1' - if section: data['section'] = section + if minor: + data['minor'] = '1' + if not minor: + data['notminor'] = '1' + if self.last_rev_time: + data['basetimestamp'] = time.strftime('%Y%m%d%H%M%S', self.last_rev_time) + if self.edit_time: + data['starttimestamp'] = time.strftime('%Y%m%d%H%M%S', self.edit_time) + if bot: + data['bot'] = '1' + if section: + data['section'] = section data.update(kwargs) def do_edit(): result = self.site.api('edit', title=self.name, text=text, - summary=summary, token=self.get_token('edit'), - **data) + summary=summary, token=self.get_token('edit'), + **data) if result['edit'].get('result').lower() == 'failure': raise errors.EditError(self, result['edit']) return result @@ -195,7 +209,7 @@ class Page(object): if e.code == 'editconflict': raise errors.EditError(self, summary, e.info) elif e.code in ('protectedtitle', 'cantcreate', 'cantcreate-anon', 'noimageredirect-anon', - 'noimageredirect', 'noedit-anon', 'noedit'): + 'noimageredirect', 'noedit-anon', 'noedit'): raise errors.ProtectedPageError(self, e.code, e.info) else: raise @@ -219,20 +233,22 @@ class Page(object): exception is raised. """ - if not self.can('move'): raise errors.InsufficientPermission(self) + if not self.can('move'): + raise errors.InsufficientPermission(self) if not self.site.writeapi: return OldPage.move(self, new_title=new_title, - reason=reason, move_talk=move_talk) + reason=reason, move_talk=move_talk) data = {} - if move_talk: data['movetalk'] = '1' - if no_redirect: data['noredirect'] = '1' + if move_talk: + data['movetalk'] = '1' + if no_redirect: + data['noredirect'] = '1' result = self.site.api('move', ('from', self.name), to=new_title, - token=self.get_token('move'), reason=reason, **data) + token=self.get_token('move'), reason=reason, **data) return result['move'] - def delete(self, reason='', watch=False, unwatch=False, oldimage=False): """Delete page. @@ -240,18 +256,22 @@ class Page(object): exception is raised. """ - if not self.can('delete'): raise errors.InsufficientPermission(self) + if not self.can('delete'): + raise errors.InsufficientPermission(self) if not self.site.writeapi: return OldPage.delete(self, reason=reason) data = {} - if watch: data['watch'] = '1' - if unwatch: data['unwatch'] = '1' - if oldimage: data['oldimage'] = oldimage + if watch: + data['watch'] = '1' + if unwatch: + data['unwatch'] = '1' + if oldimage: + data['oldimage'] = oldimage result = self.site.api('delete', title=self.name, - token=self.get_token('delete'), - reason=reason, **data) + token=self.get_token('delete'), + reason=reason, **data) return result['delete'] def purge(self): @@ -269,8 +289,9 @@ class Page(object): # Fix title for < 1.11 !! prefix = listing.List.get_prefix('bl', generator) kwargs = dict(listing.List.generate_kwargs(prefix, - namespace=namespace, filterredir=filterredir)) - if redirect: kwargs['%sredirect' % prefix] = '1' + namespace=namespace, filterredir=filterredir)) + if redirect: + kwargs['%sredirect' % prefix] = '1' kwargs[compatibility.title(prefix, self.site.require(1, 11, raise_error=False))] = self.name return listing.List.get_list(generator)(self.site, 'backlinks', 'bl', limit=limit, return_values='title', **kwargs) @@ -288,8 +309,9 @@ class Page(object): # Fix title for < 1.11 !! prefix = listing.List.get_prefix('ei', generator) kwargs = dict(listing.List.generate_kwargs(prefix, - namespace=namespace, filterredir=filterredir)) - if redirect: kwargs['%sredirect' % prefix] = '1' + namespace=namespace, filterredir=filterredir)) + if redirect: + kwargs['%sredirect' % prefix] = '1' kwargs[compatibility.title(prefix, self.site.require(1, 11, raise_error=False))] = self.name return listing.List.get_list(generator)(self.site, 'embeddedin', 'ei', limit=limit, return_values='title', **kwargs) @@ -312,22 +334,25 @@ class Page(object): def links(self, namespace=None, generator=True, redirects=False): self.site.require(1, 9) kwargs = dict(listing.List.generate_kwargs('pl', namespace=namespace)) - if redirects: kwargs['redirects'] = '1' + if redirects: + kwargs['redirects'] = '1' if generator: return listing.PagePropertyGenerator(self, 'links', 'pl', **kwargs) else: return listing.PageProperty(self, 'links', 'pl', return_values='title', **kwargs) def revisions(self, startid=None, endid=None, start=None, end=None, - dir='older', user=None, excludeuser=None, limit=50, - prop='ids|timestamp|flags|comment|user', expandtemplates=False, section=None): + dir='older', user=None, excludeuser=None, limit=50, + prop='ids|timestamp|flags|comment|user', expandtemplates=False, section=None): self.site.require(1, 8) kwargs = dict(listing.List.generate_kwargs('rv', startid=startid, endid=endid, - start=start, end=end, user=user, excludeuser=excludeuser)) + start=start, end=end, user=user, excludeuser=excludeuser)) kwargs['rvdir'] = dir kwargs['rvprop'] = prop - if expandtemplates: kwargs['rvexpandtemplates'] = '1' - if section: kwargs['rvsection'] = section + if expandtemplates: + kwargs['rvexpandtemplates'] = '1' + if section: + kwargs['rvsection'] = section return listing.RevisionsIterator(self, 'revisions', 'rv', limit=limit, **kwargs) @@ -339,34 +364,37 @@ class Page(object): else: return listing.PageProperty(self, 'templates', 'tl', return_values='title') + class Image(Page): + def __init__(self, site, name, info=None): site.require(1, 11) Page.__init__(self, site, name, info, - extra_properties={'imageinfo': (('iiprop', - compatibility.iiprop(site.version)), )}) + extra_properties={'imageinfo': (('iiprop', + compatibility.iiprop(site.version)), )}) self.imagerepository = self._info.get('imagerepository', '') self.imageinfo = self._info.get('imageinfo', ({}, ))[0] def imagehistory(self): return listing.PageProperty(self, 'imageinfo', 'ii', - iiprop=compatibility.iiprop(self.site.version)) + iiprop=compatibility.iiprop(self.site.version)) def imageusage(self, namespace=None, filterredir='all', redirect=False, - limit=None, generator=True): + limit=None, generator=True): self.site.require(1, 11) # TODO: Fix for versions < 1.11 prefix = listing.List.get_prefix('iu', generator) kwargs = dict(listing.List.generate_kwargs(prefix, title=self.name, - namespace=namespace, filterredir=filterredir)) - if redirect: kwargs['%sredirect' % prefix] = '1' + namespace=namespace, filterredir=filterredir)) + if redirect: + kwargs['%sredirect' % prefix] = '1' return listing.List.get_list(generator)(self.site, 'imageusage', 'iu', - limit=limit, return_values='title', **kwargs) + limit=limit, return_values='title', **kwargs) def duplicatefiles(self, limit=None): self.require(1, 14) return listing.PageProperty(self, 'duplicatefiles', 'df', - dflimit=limit) + dflimit=limit) def download(self): url = self.imageinfo['url'] diff --git a/mwclient/page_nowriteapi.py b/mwclient/page_nowriteapi.py index 2373b46..cfca403 100644 --- a/mwclient/page_nowriteapi.py +++ b/mwclient/page_nowriteapi.py @@ -4,7 +4,9 @@ from htmlentitydefs import name2codepoint import errors + class OldPage(object): + @staticmethod def save(self, text=u'', summary=u'', minor=False): data = {} @@ -22,7 +24,8 @@ class OldPage(object): data['wpStarttime'] = time.strftime('%Y%m%d%H%M%S', time.gmtime()) data['wpStarttime'] = time.strftime('%Y%m%d%H%M%S', time.gmtime()) - if minor: data['wpMinoredit'] = '1' + if minor: + data['wpMinoredit'] = '1' data['title'] = self.name page_data = self.site.raw_index('submit', **data) @@ -32,18 +35,20 @@ class OldPage(object): page.close() if page.data: - if page.readonly: raise errors.ProtectedPageError(self) + if page.readonly: + raise errors.ProtectedPageError(self) self.get_token('edit', True) raise errors.EditError(page.title, data) @staticmethod def move(self, new_title, reason='', move_talk=True): postdata = {'wpNewTitle': new_title, - 'wpOldTitle': self.name, - 'wpReason': reason, - 'wpMove': '1', - 'wpEditToken': self.get_token('move')} - if move_talk: postdata['wpMovetalk'] = '1' + 'wpOldTitle': self.name, + 'wpReason': reason, + 'wpMove': '1', + 'wpEditToken': self.get_token('move')} + if move_talk: + postdata['wpMovetalk'] = '1' postdata['title'] = 'Special:Movepage' page_data = self.site.raw_index('submit', **data) @@ -58,14 +63,16 @@ class OldPage(object): @staticmethod def delete(self, reason=''): postdata = {'wpReason': reason, - 'wpConfirmB': 'Delete', - 'mw-filedelete-submit': 'Delete', - 'wpEditToken': self.get_token('delete'), - 'title': self.name} + 'wpConfirmB': 'Delete', + 'mw-filedelete-submit': 'Delete', + 'wpEditToken': self.get_token('delete'), + 'title': self.name} page_data = self.site.raw_index('delete', **postdata) + class EditPage(HTMLParser): + def __init__(self, form): HTMLParser.__init__(self) @@ -92,27 +99,33 @@ class EditPage(HTMLParser): if tag == 'input' and self.in_form and (u'type', u'submit') \ not in attrs and (u'type', u'checkbox') not in attrs: attrs = dict(attrs) - if u'name' in attrs: self.data[attrs[u'name']] = attrs.get(u'value', u'') + if u'name' in attrs: + self.data[attrs[u'name']] = attrs.get(u'value', u'') if self.in_form and tag == 'textarea': self.in_text = True self.readonly = (u'readonly', u'readonly') in attrs - def handle_endtag(self, tag): - if self.in_title and tag == 'title': self.in_title = False - if self.in_form and tag == 'form': self.in_form = False - if self.in_text and tag == 'textarea': self.in_text = False + if self.in_title and tag == 'title': + self.in_title = False + if self.in_form and tag == 'form': + self.in_form = False + if self.in_text and tag == 'textarea': + self.in_text = False def handle_data(self, data): - if self.in_text: self.textdata.append(data) - if self.in_title: self.title += data + if self.in_text: + self.textdata.append(data) + if self.in_title: + self.title += data def handle_entityref(self, name): if name in name2codepoint: self.handle_data(unichr(name2codepoint[name])) else: self.handle_data(u'&%s;' % name) + def handle_charref(self, name): try: self.handle_data(unichr(int(name))) diff --git a/mwclient/upload.py b/mwclient/upload.py index ec82edf..4d79cc2 100644 --- a/mwclient/upload.py +++ b/mwclient/upload.py @@ -1,7 +1,9 @@ import random from cStringIO import StringIO + class Upload(object): + """ Base class for upload objects. This class should always be subclassed by upload classes and its constructor always be called. @@ -11,12 +13,14 @@ class Upload(object): """ BLOCK_SIZE = 8192 + def __init__(self, length, content_type): self.length = length self.content_type = content_type def __iter__(self): return self + def next(self): data = self.read(self.BLOCK_SIZE) if data == '': @@ -32,27 +36,35 @@ class Upload(object): else: return s + class UploadRawData(Upload): + """ This upload class is simply a wrapper around StringIO """ + def __init__(self, data, content_type='application/x-www-form-urlencoded'): self.fstr = StringIO(data) Upload.__init__(self, len(data), content_type) + def read(self, length=-1): return self.fstr.read(length) class UploadDict(UploadRawData): + """ This class creates an x-www-form-urlencoded representation of a dict and then passes it through its parent UploadRawData """ + def __init__(self, data): postdata = '&'.join('%s=%s' % (self.encode(i), self.encode(data[i])) for i in data) UploadRawData.__init__(self, postdata) + class UploadFile(Upload): + """ This class accepts a file with information and a postdata dictionary and creates a multipart/form-data representation from it. @@ -62,21 +74,22 @@ class UploadFile(Upload): STAGE_POSTDATA = 2 STAGE_FOOTER = 3 STAGE_DONE = 4 + def __init__(self, filefield, filename, filelength, file, data): - self.stage = self.STAGE_FILEHEADER; + self.stage = self.STAGE_FILEHEADER self.boundary = self.generate_boundary() self.postdata = self.generate_multipart_from_dict(data) self.footer = '\r\n--%s--\r\n' % self.boundary self.fileheader = ('--%s\r\n' % self.boundary + - 'Content-Disposition: form-data; name="%s"; filename="%s"\r\n' % - (self.encode(filefield), self.encode(filename)) + - 'Content-Type: application/octet-stream\r\n\r\n') + 'Content-Disposition: form-data; name="%s"; filename="%s"\r\n' % + (self.encode(filefield), self.encode(filename)) + + 'Content-Type: application/octet-stream\r\n\r\n') self.file = file self.length_left = filelength self.str_data = None Upload.__init__(self, len(self.fileheader) + filelength + len(self.postdata) + len(self.footer) + 2, - 'multipart/form-data; boundary=' + self.boundary) + 'multipart/form-data; boundary=' + self.boundary) def read(self, length): if self.stage == self.STAGE_DONE: @@ -106,7 +119,6 @@ class UploadFile(Upload): return self.read(length) return data - @staticmethod def generate_boundary(): return '----%s----' % ''.join((random.choice( -- GitLab