From 4c43c9724c25c3d919cb535973559fab1f1c0a7b Mon Sep 17 00:00:00 2001 From: Benjamin Mako Hill Date: Fri, 25 Apr 2014 16:50:55 -0700 Subject: [PATCH] initial version of several example programs using the tweepy twitter API --- .gitignore | 3 + README | 21 ++ docs/oath-LICENSE.txt | 21 ++ docs/oauth-PKG-INFO | 10 + docs/tweepy-LICENSE | 20 ++ docs/tweepy-README.md | 33 ++ oauth/._oauth.py | Bin 0 -> 167 bytes oauth/__init__.py | 0 oauth/example/._server.py | Bin 0 -> 167 bytes oauth/example/client.py | 165 +++++++++ oauth/example/server.py | 195 ++++++++++ oauth/oauth.py | 655 +++++++++++++++++++++++++++++++++ tweepy/__init__.py | 27 ++ tweepy/api.py | 740 ++++++++++++++++++++++++++++++++++++++ tweepy/auth.py | 156 ++++++++ tweepy/binder.py | 213 +++++++++++ tweepy/cache.py | 424 ++++++++++++++++++++++ tweepy/cursor.py | 171 +++++++++ tweepy/error.py | 15 + tweepy/models.py | 436 ++++++++++++++++++++++ tweepy/oauth.py | 655 +++++++++++++++++++++++++++++++++ tweepy/parsers.py | 97 +++++ tweepy/streaming.py | 319 ++++++++++++++++ tweepy/utils.py | 59 +++ twitter-stream1.py | 20 ++ twitter-stream2.py | 21 ++ twitter1.py | 12 + twitter2.py | 17 + twitter3.py | 12 + twitter4.py | 17 + twitter_authentication.py | 4 + 31 files changed, 4538 insertions(+) create mode 100644 .gitignore create mode 100644 README create mode 100644 docs/oath-LICENSE.txt create mode 100644 docs/oauth-PKG-INFO create mode 100644 docs/tweepy-LICENSE create mode 100644 docs/tweepy-README.md create mode 100644 oauth/._oauth.py create mode 100644 oauth/__init__.py create mode 100644 oauth/example/._server.py create mode 100644 oauth/example/client.py create mode 100644 oauth/example/server.py create mode 100644 oauth/oauth.py create mode 100644 tweepy/__init__.py create mode 100644 tweepy/api.py create mode 100644 tweepy/auth.py create mode 100644 tweepy/binder.py create mode 100644 tweepy/cache.py create mode 100644 tweepy/cursor.py create mode 100644 tweepy/error.py create mode 100644 tweepy/models.py create mode 100644 tweepy/oauth.py create mode 100644 tweepy/parsers.py create mode 100644 tweepy/streaming.py create mode 100644 tweepy/utils.py create mode 100644 twitter-stream1.py create mode 100644 twitter-stream2.py create mode 100644 twitter1.py create mode 100644 twitter2.py create mode 100644 twitter3.py create mode 100644 twitter4.py create mode 100644 twitter_authentication.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..0bc2e1e --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +*~ +*.pyc +MY_DATA.tsv diff --git a/README b/README new file mode 100644 index 0000000..37c50fb --- /dev/null +++ b/README @@ -0,0 +1,21 @@ +To use these example scripts, you will need authentication details +from Twitter. + +1. + +First, go create and retrieve authenication information Twitter. +Details on how to create this are all on this page: + +https://openhatch.org/wiki/Community_Data_Science_Workshops/Twitter + +2. + +Edit the file twitter_authentication.py in this directory to record +this information. There are four pieces of information you need and +you will replace every "CHANGE ME" with the value that Twitter has +given you: + +API_KEY = 'CHANGE_ME' +API_SECRET = 'CHANGE_ME' +ACCESS_TOKEN = 'CHANGE_ME' +ACCESS_TOKEN_SECRET = 'CHANGE_ME' diff --git a/docs/oath-LICENSE.txt b/docs/oath-LICENSE.txt new file mode 100644 index 0000000..8f5e8bd --- /dev/null +++ b/docs/oath-LICENSE.txt @@ -0,0 +1,21 @@ +The MIT License + +Copyright (c) 2007 Leah Culver + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. \ No newline at end of file diff --git a/docs/oauth-PKG-INFO b/docs/oauth-PKG-INFO new file mode 100644 index 0000000..c27aaff --- /dev/null +++ b/docs/oauth-PKG-INFO @@ -0,0 +1,10 @@ +Metadata-Version: 1.0 +Name: oauth +Version: 1.0.1 +Summary: Library for OAuth version 1.0a. +Home-page: http://code.google.com/p/oauth +Author: Leah Culver +Author-email: leah.culver@gmail.com +License: MIT License +Description: UNKNOWN +Platform: UNKNOWN diff --git a/docs/tweepy-LICENSE b/docs/tweepy-LICENSE new file mode 100644 index 0000000..545a75c --- /dev/null +++ b/docs/tweepy-LICENSE @@ -0,0 +1,20 @@ +MIT License +Copyright (c) 2013-2014 Joshua Roesslein + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/docs/tweepy-README.md b/docs/tweepy-README.md new file mode 100644 index 0000000..cf54aae --- /dev/null +++ b/docs/tweepy-README.md @@ -0,0 +1,33 @@ +Tweepy: Twitter for Python! +====== +[![Build Status](https://travis-ci.org/tweepy/tweepy.png?branch=master)](https://travis-ci.org/tweepy/tweepy) +[![Downloads](https://pypip.in/d/tweepy/badge.png)](https://crate.io/packages/tweepy) [![Downloads](https://pypip.in/v/tweepy/badge.png)](https://crate.io/packages/tweepy) +[![Coverage Status](https://coveralls.io/repos/tweepy/tweepy/badge.png?branch=master)](https://coveralls.io/r/tweepy/tweepy?branch=master) + +Installation +------------ +The easiest way to install the latest version +is by using pip/easy_install to pull it from PyPI: + + pip install tweepy + +You may also use Git to clone the repository from +Github and install it manually: + + git clone https://github.com/tweepy/tweepy.git + python setup.py install + +**Note** only Python 2.6 and 2.7 are supported at +the moment. The Python 3 family is not yet supported. + +Documentation +------------- + - [Website (Work in-progress)](http://tweepy.github.com/) + - [Twitter Developers](http://dev.twitter.com/) + - [Python Package Documentation](http://packages.python.org/tweepy/html/index.html) + +Community +--------- + - [Google Group/Mailing list](http://groups.google.com/group/tweepy) + - IRC Chat (Freenode.net #tweepy) + diff --git a/oauth/._oauth.py b/oauth/._oauth.py new file mode 100644 index 0000000000000000000000000000000000000000..02899ca2be0d1044e2ae7ca2d3e1c6d87e791c8d GIT binary patch literal 167 zcmZQz6=P>$Vqox1Ojhs@R)|o50+1L3ClDI}aVZdk_{)Kq2;dkJ62!2jP6((Jq+<>c vbE9cvM2L$g=jZAr78K;9>V>3Ml(^<4=ci=mr2`fCIy!s$h4=@1SQ`KU=XMm~ literal 0 HcmV?d00001 diff --git a/oauth/__init__.py b/oauth/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/oauth/example/._server.py b/oauth/example/._server.py new file mode 100644 index 0000000000000000000000000000000000000000..f4dda4ed87ef860ac52f8232bf9fe92230ca9fd0 GIT binary patch literal 167 zcmZQz6=P>$Vqox1Ojhs@R)|o50+1L3ClDI}aVZdk_{)Kq2;dkJ62!2PLj$N3q+<>c vbE9cvM2L$g=jZAr78K;9>V>3Ml(^<4=ci=mr2`fCIy!s$h4=@1SQ`KU$(9s; literal 0 HcmV?d00001 diff --git a/oauth/example/client.py b/oauth/example/client.py new file mode 100644 index 0000000..34f7dcb --- /dev/null +++ b/oauth/example/client.py @@ -0,0 +1,165 @@ +""" +The MIT License + +Copyright (c) 2007 Leah Culver + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +Example consumer. This is not recommended for production. +Instead, you'll want to create your own subclass of OAuthClient +or find one that works with your web framework. +""" + +import httplib +import time +import oauth.oauth as oauth + +# settings for the local test consumer +SERVER = 'localhost' +PORT = 8080 + +# fake urls for the test server (matches ones in server.py) +REQUEST_TOKEN_URL = 'https://photos.example.net/request_token' +ACCESS_TOKEN_URL = 'https://photos.example.net/access_token' +AUTHORIZATION_URL = 'https://photos.example.net/authorize' +CALLBACK_URL = 'http://printer.example.com/request_token_ready' +RESOURCE_URL = 'http://photos.example.net/photos' + +# key and secret granted by the service provider for this consumer application - same as the MockOAuthDataStore +CONSUMER_KEY = 'key' +CONSUMER_SECRET = 'secret' + +# example client using httplib with headers +class SimpleOAuthClient(oauth.OAuthClient): + + def __init__(self, server, port=httplib.HTTP_PORT, request_token_url='', access_token_url='', authorization_url=''): + self.server = server + self.port = port + self.request_token_url = request_token_url + self.access_token_url = access_token_url + self.authorization_url = authorization_url + self.connection = httplib.HTTPConnection("%s:%d" % (self.server, self.port)) + + def fetch_request_token(self, oauth_request): + # via headers + # -> OAuthToken + self.connection.request(oauth_request.http_method, self.request_token_url, headers=oauth_request.to_header()) + response = self.connection.getresponse() + return oauth.OAuthToken.from_string(response.read()) + + def fetch_access_token(self, oauth_request): + # via headers + # -> OAuthToken + self.connection.request(oauth_request.http_method, self.access_token_url, headers=oauth_request.to_header()) + response = self.connection.getresponse() + return oauth.OAuthToken.from_string(response.read()) + + def authorize_token(self, oauth_request): + # via url + # -> typically just some okay response + self.connection.request(oauth_request.http_method, oauth_request.to_url()) + response = self.connection.getresponse() + return response.read() + + def access_resource(self, oauth_request): + # via post body + # -> some protected resources + headers = {'Content-Type' :'application/x-www-form-urlencoded'} + self.connection.request('POST', RESOURCE_URL, body=oauth_request.to_postdata(), headers=headers) + response = self.connection.getresponse() + return response.read() + +def run_example(): + + # setup + print '** OAuth Python Library Example **' + client = SimpleOAuthClient(SERVER, PORT, REQUEST_TOKEN_URL, ACCESS_TOKEN_URL, AUTHORIZATION_URL) + consumer = oauth.OAuthConsumer(CONSUMER_KEY, CONSUMER_SECRET) + signature_method_plaintext = oauth.OAuthSignatureMethod_PLAINTEXT() + signature_method_hmac_sha1 = oauth.OAuthSignatureMethod_HMAC_SHA1() + pause() + + # get request token + print '* Obtain a request token ...' + pause() + oauth_request = oauth.OAuthRequest.from_consumer_and_token(consumer, callback=CALLBACK_URL, http_url=client.request_token_url) + oauth_request.sign_request(signature_method_plaintext, consumer, None) + print 'REQUEST (via headers)' + print 'parameters: %s' % str(oauth_request.parameters) + pause() + token = client.fetch_request_token(oauth_request) + print 'GOT' + print 'key: %s' % str(token.key) + print 'secret: %s' % str(token.secret) + print 'callback confirmed? %s' % str(token.callback_confirmed) + pause() + + print '* Authorize the request token ...' + pause() + oauth_request = oauth.OAuthRequest.from_token_and_callback(token=token, http_url=client.authorization_url) + print 'REQUEST (via url query string)' + print 'parameters: %s' % str(oauth_request.parameters) + pause() + # this will actually occur only on some callback + response = client.authorize_token(oauth_request) + print 'GOT' + print response + # sad way to get the verifier + import urlparse, cgi + query = urlparse.urlparse(response)[4] + params = cgi.parse_qs(query, keep_blank_values=False) + verifier = params['oauth_verifier'][0] + print 'verifier: %s' % verifier + pause() + + # get access token + print '* Obtain an access token ...' + pause() + oauth_request = oauth.OAuthRequest.from_consumer_and_token(consumer, token=token, verifier=verifier, http_url=client.access_token_url) + oauth_request.sign_request(signature_method_plaintext, consumer, token) + print 'REQUEST (via headers)' + print 'parameters: %s' % str(oauth_request.parameters) + pause() + token = client.fetch_access_token(oauth_request) + print 'GOT' + print 'key: %s' % str(token.key) + print 'secret: %s' % str(token.secret) + pause() + + # access some protected resources + print '* Access protected resources ...' + pause() + parameters = {'file': 'vacation.jpg', 'size': 'original'} # resource specific params + oauth_request = oauth.OAuthRequest.from_consumer_and_token(consumer, token=token, http_method='POST', http_url=RESOURCE_URL, parameters=parameters) + oauth_request.sign_request(signature_method_hmac_sha1, consumer, token) + print 'REQUEST (via post body)' + print 'parameters: %s' % str(oauth_request.parameters) + pause() + params = client.access_resource(oauth_request) + print 'GOT' + print 'non-oauth parameters: %s' % params + pause() + +def pause(): + print '' + time.sleep(1) + +if __name__ == '__main__': + run_example() + print 'Done.' \ No newline at end of file diff --git a/oauth/example/server.py b/oauth/example/server.py new file mode 100644 index 0000000..5986b0e --- /dev/null +++ b/oauth/example/server.py @@ -0,0 +1,195 @@ +""" +The MIT License + +Copyright (c) 2007 Leah Culver + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +""" + +from BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer +import urllib + +import oauth.oauth as oauth + +# fake urls for the test server +REQUEST_TOKEN_URL = 'https://photos.example.net/request_token' +ACCESS_TOKEN_URL = 'https://photos.example.net/access_token' +AUTHORIZATION_URL = 'https://photos.example.net/authorize' +CALLBACK_URL = 'http://printer.example.com/request_token_ready' +RESOURCE_URL = 'http://photos.example.net/photos' +REALM = 'http://photos.example.net/' +VERIFIER = 'verifier' + +# example store for one of each thing +class MockOAuthDataStore(oauth.OAuthDataStore): + + def __init__(self): + self.consumer = oauth.OAuthConsumer('key', 'secret') + self.request_token = oauth.OAuthToken('requestkey', 'requestsecret') + self.access_token = oauth.OAuthToken('accesskey', 'accesssecret') + self.nonce = 'nonce' + self.verifier = VERIFIER + + def lookup_consumer(self, key): + if key == self.consumer.key: + return self.consumer + return None + + def lookup_token(self, token_type, token): + token_attrib = getattr(self, '%s_token' % token_type) + if token == token_attrib.key: + ## HACK + token_attrib.set_callback(CALLBACK_URL) + return token_attrib + return None + + def lookup_nonce(self, oauth_consumer, oauth_token, nonce): + if oauth_token and oauth_consumer.key == self.consumer.key and (oauth_token.key == self.request_token.key or oauth_token.key == self.access_token.key) and nonce == self.nonce: + return self.nonce + return None + + def fetch_request_token(self, oauth_consumer, oauth_callback): + if oauth_consumer.key == self.consumer.key: + if oauth_callback: + # want to check here if callback is sensible + # for mock store, we assume it is + self.request_token.set_callback(oauth_callback) + return self.request_token + return None + + def fetch_access_token(self, oauth_consumer, oauth_token, oauth_verifier): + if oauth_consumer.key == self.consumer.key and oauth_token.key == self.request_token.key and oauth_verifier == self.verifier: + # want to check here if token is authorized + # for mock store, we assume it is + return self.access_token + return None + + def authorize_request_token(self, oauth_token, user): + if oauth_token.key == self.request_token.key: + # authorize the request token in the store + # for mock store, do nothing + return self.request_token + return None + +class RequestHandler(BaseHTTPRequestHandler): + + def __init__(self, *args, **kwargs): + self.oauth_server = oauth.OAuthServer(MockOAuthDataStore()) + self.oauth_server.add_signature_method(oauth.OAuthSignatureMethod_PLAINTEXT()) + self.oauth_server.add_signature_method(oauth.OAuthSignatureMethod_HMAC_SHA1()) + BaseHTTPRequestHandler.__init__(self, *args, **kwargs) + + # example way to send an oauth error + def send_oauth_error(self, err=None): + # send a 401 error + self.send_error(401, str(err.message)) + # return the authenticate header + header = oauth.build_authenticate_header(realm=REALM) + for k, v in header.iteritems(): + self.send_header(k, v) + + def do_GET(self): + + # debug info + #print self.command, self.path, self.headers + + # get the post data (if any) + postdata = None + if self.command == 'POST': + try: + length = int(self.headers.getheader('content-length')) + postdata = self.rfile.read(length) + except: + pass + + # construct the oauth request from the request parameters + oauth_request = oauth.OAuthRequest.from_request(self.command, self.path, headers=self.headers, query_string=postdata) + + # request token + if self.path.startswith(REQUEST_TOKEN_URL): + try: + # create a request token + token = self.oauth_server.fetch_request_token(oauth_request) + # send okay response + self.send_response(200, 'OK') + self.end_headers() + # return the token + self.wfile.write(token.to_string()) + except oauth.OAuthError, err: + self.send_oauth_error(err) + return + + # user authorization + if self.path.startswith(AUTHORIZATION_URL): + try: + # get the request token + token = self.oauth_server.fetch_request_token(oauth_request) + # authorize the token (kind of does nothing for now) + token = self.oauth_server.authorize_token(token, None) + token.set_verifier(VERIFIER) + # send okay response + self.send_response(200, 'OK') + self.end_headers() + # return the callback url (to show server has it) + self.wfile.write(token.get_callback_url()) + except oauth.OAuthError, err: + self.send_oauth_error(err) + return + + # access token + if self.path.startswith(ACCESS_TOKEN_URL): + try: + # create an access token + token = self.oauth_server.fetch_access_token(oauth_request) + # send okay response + self.send_response(200, 'OK') + self.end_headers() + # return the token + self.wfile.write(token.to_string()) + except oauth.OAuthError, err: + self.send_oauth_error(err) + return + + # protected resources + if self.path.startswith(RESOURCE_URL): + try: + # verify the request has been oauth authorized + consumer, token, params = self.oauth_server.verify_request(oauth_request) + # send okay response + self.send_response(200, 'OK') + self.end_headers() + # return the extra parameters - just for something to return + self.wfile.write(str(params)) + except oauth.OAuthError, err: + self.send_oauth_error(err) + return + + def do_POST(self): + return self.do_GET() + +def main(): + try: + server = HTTPServer(('', 8080), RequestHandler) + print 'Test server running...' + server.serve_forever() + except KeyboardInterrupt: + server.socket.close() + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/oauth/oauth.py b/oauth/oauth.py new file mode 100644 index 0000000..b6284c5 --- /dev/null +++ b/oauth/oauth.py @@ -0,0 +1,655 @@ +""" +The MIT License + +Copyright (c) 2007 Leah Culver + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +""" + +import cgi +import urllib +import time +import random +import urlparse +import hmac +import binascii + + +VERSION = '1.0' # Hi Blaine! +HTTP_METHOD = 'GET' +SIGNATURE_METHOD = 'PLAINTEXT' + + +class OAuthError(RuntimeError): + """Generic exception class.""" + def __init__(self, message='OAuth error occured.'): + self.message = message + +def build_authenticate_header(realm=''): + """Optional WWW-Authenticate header (401 error)""" + return {'WWW-Authenticate': 'OAuth realm="%s"' % realm} + +def escape(s): + """Escape a URL including any /.""" + return urllib.quote(s, safe='~') + +def _utf8_str(s): + """Convert unicode to utf-8.""" + if isinstance(s, unicode): + return s.encode("utf-8") + else: + return str(s) + +def generate_timestamp(): + """Get seconds since epoch (UTC).""" + return int(time.time()) + +def generate_nonce(length=8): + """Generate pseudorandom number.""" + return ''.join([str(random.randint(0, 9)) for i in range(length)]) + +def generate_verifier(length=8): + """Generate pseudorandom number.""" + return ''.join([str(random.randint(0, 9)) for i in range(length)]) + + +class OAuthConsumer(object): + """Consumer of OAuth authentication. + + OAuthConsumer is a data type that represents the identity of the Consumer + via its shared secret with the Service Provider. + + """ + key = None + secret = None + + def __init__(self, key, secret): + self.key = key + self.secret = secret + + +class OAuthToken(object): + """OAuthToken is a data type that represents an End User via either an access + or request token. + + key -- the token + secret -- the token secret + + """ + key = None + secret = None + callback = None + callback_confirmed = None + verifier = None + + def __init__(self, key, secret): + self.key = key + self.secret = secret + + def set_callback(self, callback): + self.callback = callback + self.callback_confirmed = 'true' + + def set_verifier(self, verifier=None): + if verifier is not None: + self.verifier = verifier + else: + self.verifier = generate_verifier() + + def get_callback_url(self): + if self.callback and self.verifier: + # Append the oauth_verifier. + parts = urlparse.urlparse(self.callback) + scheme, netloc, path, params, query, fragment = parts[:6] + if query: + query = '%s&oauth_verifier=%s' % (query, self.verifier) + else: + query = 'oauth_verifier=%s' % self.verifier + return urlparse.urlunparse((scheme, netloc, path, params, + query, fragment)) + return self.callback + + def to_string(self): + data = { + 'oauth_token': self.key, + 'oauth_token_secret': self.secret, + } + if self.callback_confirmed is not None: + data['oauth_callback_confirmed'] = self.callback_confirmed + return urllib.urlencode(data) + + def from_string(s): + """ Returns a token from something like: + oauth_token_secret=xxx&oauth_token=xxx + """ + params = cgi.parse_qs(s, keep_blank_values=False) + key = params['oauth_token'][0] + secret = params['oauth_token_secret'][0] + token = OAuthToken(key, secret) + try: + token.callback_confirmed = params['oauth_callback_confirmed'][0] + except KeyError: + pass # 1.0, no callback confirmed. + return token + from_string = staticmethod(from_string) + + def __str__(self): + return self.to_string() + + +class OAuthRequest(object): + """OAuthRequest represents the request and can be serialized. + + OAuth parameters: + - oauth_consumer_key + - oauth_token + - oauth_signature_method + - oauth_signature + - oauth_timestamp + - oauth_nonce + - oauth_version + - oauth_verifier + ... any additional parameters, as defined by the Service Provider. + """ + parameters = None # OAuth parameters. + http_method = HTTP_METHOD + http_url = None + version = VERSION + + def __init__(self, http_method=HTTP_METHOD, http_url=None, parameters=None): + self.http_method = http_method + self.http_url = http_url + self.parameters = parameters or {} + + def set_parameter(self, parameter, value): + self.parameters[parameter] = value + + def get_parameter(self, parameter): + try: + return self.parameters[parameter] + except: + raise OAuthError('Parameter not found: %s' % parameter) + + def _get_timestamp_nonce(self): + return self.get_parameter('oauth_timestamp'), self.get_parameter( + 'oauth_nonce') + + def get_nonoauth_parameters(self): + """Get any non-OAuth parameters.""" + parameters = {} + for k, v in self.parameters.iteritems(): + # Ignore oauth parameters. + if k.find('oauth_') < 0: + parameters[k] = v + return parameters + + def to_header(self, realm=''): + """Serialize as a header for an HTTPAuth request.""" + auth_header = 'OAuth realm="%s"' % realm + # Add the oauth parameters. + if self.parameters: + for k, v in self.parameters.iteritems(): + if k[:6] == 'oauth_': + auth_header += ', %s="%s"' % (k, escape(str(v))) + return {'Authorization': auth_header} + + def to_postdata(self): + """Serialize as post data for a POST request.""" + return '&'.join(['%s=%s' % (escape(str(k)), escape(str(v))) \ + for k, v in self.parameters.iteritems()]) + + def to_url(self): + """Serialize as a URL for a GET request.""" + return '%s?%s' % (self.get_normalized_http_url(), self.to_postdata()) + + def get_normalized_parameters(self): + """Return a string that contains the parameters that must be signed.""" + params = self.parameters + try: + # Exclude the signature if it exists. + del params['oauth_signature'] + except: + pass + # Escape key values before sorting. + key_values = [(escape(_utf8_str(k)), escape(_utf8_str(v))) \ + for k,v in params.items()] + # Sort lexicographically, first after key, then after value. + key_values.sort() + # Combine key value pairs into a string. + return '&'.join(['%s=%s' % (k, v) for k, v in key_values]) + + def get_normalized_http_method(self): + """Uppercases the http method.""" + return self.http_method.upper() + + def get_normalized_http_url(self): + """Parses the URL and rebuilds it to be scheme://host/path.""" + parts = urlparse.urlparse(self.http_url) + scheme, netloc, path = parts[:3] + # Exclude default port numbers. + if scheme == 'http' and netloc[-3:] == ':80': + netloc = netloc[:-3] + elif scheme == 'https' and netloc[-4:] == ':443': + netloc = netloc[:-4] + return '%s://%s%s' % (scheme, netloc, path) + + def sign_request(self, signature_method, consumer, token): + """Set the signature parameter to the result of build_signature.""" + # Set the signature method. + self.set_parameter('oauth_signature_method', + signature_method.get_name()) + # Set the signature. + self.set_parameter('oauth_signature', + self.build_signature(signature_method, consumer, token)) + + def build_signature(self, signature_method, consumer, token): + """Calls the build signature method within the signature method.""" + return signature_method.build_signature(self, consumer, token) + + def from_request(http_method, http_url, headers=None, parameters=None, + query_string=None): + """Combines multiple parameter sources.""" + if parameters is None: + parameters = {} + + # Headers + if headers and 'Authorization' in headers: + auth_header = headers['Authorization'] + # Check that the authorization header is OAuth. + if auth_header[:6] == 'OAuth ': + auth_header = auth_header[6:] + try: + # Get the parameters from the header. + header_params = OAuthRequest._split_header(auth_header) + parameters.update(header_params) + except: + raise OAuthError('Unable to parse OAuth parameters from ' + 'Authorization header.') + + # GET or POST query string. + if query_string: + query_params = OAuthRequest._split_url_string(query_string) + parameters.update(query_params) + + # URL parameters. + param_str = urlparse.urlparse(http_url)[4] # query + url_params = OAuthRequest._split_url_string(param_str) + parameters.update(url_params) + + if parameters: + return OAuthRequest(http_method, http_url, parameters) + + return None + from_request = staticmethod(from_request) + + def from_consumer_and_token(oauth_consumer, token=None, + callback=None, verifier=None, http_method=HTTP_METHOD, + http_url=None, parameters=None): + if not parameters: + parameters = {} + + defaults = { + 'oauth_consumer_key': oauth_consumer.key, + 'oauth_timestamp': generate_timestamp(), + 'oauth_nonce': generate_nonce(), + 'oauth_version': OAuthRequest.version, + } + + defaults.update(parameters) + parameters = defaults + + if token: + parameters['oauth_token'] = token.key + if token.callback: + parameters['oauth_callback'] = token.callback + # 1.0a support for verifier. + if verifier: + parameters['oauth_verifier'] = verifier + elif callback: + # 1.0a support for callback in the request token request. + parameters['oauth_callback'] = callback + + return OAuthRequest(http_method, http_url, parameters) + from_consumer_and_token = staticmethod(from_consumer_and_token) + + def from_token_and_callback(token, callback=None, http_method=HTTP_METHOD, + http_url=None, parameters=None): + if not parameters: + parameters = {} + + parameters['oauth_token'] = token.key + + if callback: + parameters['oauth_callback'] = callback + + return OAuthRequest(http_method, http_url, parameters) + from_token_and_callback = staticmethod(from_token_and_callback) + + def _split_header(header): + """Turn Authorization: header into parameters.""" + params = {} + parts = header.split(',') + for param in parts: + # Ignore realm parameter. + if param.find('realm') > -1: + continue + # Remove whitespace. + param = param.strip() + # Split key-value. + param_parts = param.split('=', 1) + # Remove quotes and unescape the value. + params[param_parts[0]] = urllib.unquote(param_parts[1].strip('\"')) + return params + _split_header = staticmethod(_split_header) + + def _split_url_string(param_str): + """Turn URL string into parameters.""" + parameters = cgi.parse_qs(param_str, keep_blank_values=False) + for k, v in parameters.iteritems(): + parameters[k] = urllib.unquote(v[0]) + return parameters + _split_url_string = staticmethod(_split_url_string) + +class OAuthServer(object): + """A worker to check the validity of a request against a data store.""" + timestamp_threshold = 300 # In seconds, five minutes. + version = VERSION + signature_methods = None + data_store = None + + def __init__(self, data_store=None, signature_methods=None): + self.data_store = data_store + self.signature_methods = signature_methods or {} + + def set_data_store(self, data_store): + self.data_store = data_store + + def get_data_store(self): + return self.data_store + + def add_signature_method(self, signature_method): + self.signature_methods[signature_method.get_name()] = signature_method + return self.signature_methods + + def fetch_request_token(self, oauth_request): + """Processes a request_token request and returns the + request token on success. + """ + try: + # Get the request token for authorization. + token = self._get_token(oauth_request, 'request') + except OAuthError: + # No token required for the initial token request. + version = self._get_version(oauth_request) + consumer = self._get_consumer(oauth_request) + try: + callback = self.get_callback(oauth_request) + except OAuthError: + callback = None # 1.0, no callback specified. + self._check_signature(oauth_request, consumer, None) + # Fetch a new token. + token = self.data_store.fetch_request_token(consumer, callback) + return token + + def fetch_access_token(self, oauth_request): + """Processes an access_token request and returns the + access token on success. + """ + version = self._get_version(oauth_request) + consumer = self._get_consumer(oauth_request) + try: + verifier = self._get_verifier(oauth_request) + except OAuthError: + verifier = None + # Get the request token. + token = self._get_token(oauth_request, 'request') + self._check_signature(oauth_request, consumer, token) + new_token = self.data_store.fetch_access_token(consumer, token, verifier) + return new_token + + def verify_request(self, oauth_request): + """Verifies an api call and checks all the parameters.""" + # -> consumer and token + version = self._get_version(oauth_request) + consumer = self._get_consumer(oauth_request) + # Get the access token. + token = self._get_token(oauth_request, 'access') + self._check_signature(oauth_request, consumer, token) + parameters = oauth_request.get_nonoauth_parameters() + return consumer, token, parameters + + def authorize_token(self, token, user): + """Authorize a request token.""" + return self.data_store.authorize_request_token(token, user) + + def get_callback(self, oauth_request): + """Get the callback URL.""" + return oauth_request.get_parameter('oauth_callback') + + def build_authenticate_header(self, realm=''): + """Optional support for the authenticate header.""" + return {'WWW-Authenticate': 'OAuth realm="%s"' % realm} + + def _get_version(self, oauth_request): + """Verify the correct version request for this server.""" + try: + version = oauth_request.get_parameter('oauth_version') + except: + version = VERSION + if version and version != self.version: + raise OAuthError('OAuth version %s not supported.' % str(version)) + return version + + def _get_signature_method(self, oauth_request): + """Figure out the signature with some defaults.""" + try: + signature_method = oauth_request.get_parameter( + 'oauth_signature_method') + except: + signature_method = SIGNATURE_METHOD + try: + # Get the signature method object. + signature_method = self.signature_methods[signature_method] + except: + signature_method_names = ', '.join(self.signature_methods.keys()) + raise OAuthError('Signature method %s not supported try one of the ' + 'following: %s' % (signature_method, signature_method_names)) + + return signature_method + + def _get_consumer(self, oauth_request): + consumer_key = oauth_request.get_parameter('oauth_consumer_key') + consumer = self.data_store.lookup_consumer(consumer_key) + if not consumer: + raise OAuthError('Invalid consumer.') + return consumer + + def _get_token(self, oauth_request, token_type='access'): + """Try to find the token for the provided request token key.""" + token_field = oauth_request.get_parameter('oauth_token') + token = self.data_store.lookup_token(token_type, token_field) + if not token: + raise OAuthError('Invalid %s token: %s' % (token_type, token_field)) + return token + + def _get_verifier(self, oauth_request): + return oauth_request.get_parameter('oauth_verifier') + + def _check_signature(self, oauth_request, consumer, token): + timestamp, nonce = oauth_request._get_timestamp_nonce() + self._check_timestamp(timestamp) + self._check_nonce(consumer, token, nonce) + signature_method = self._get_signature_method(oauth_request) + try: + signature = oauth_request.get_parameter('oauth_signature') + except: + raise OAuthError('Missing signature.') + # Validate the signature. + valid_sig = signature_method.check_signature(oauth_request, consumer, + token, signature) + if not valid_sig: + key, base = signature_method.build_signature_base_string( + oauth_request, consumer, token) + raise OAuthError('Invalid signature. Expected signature base ' + 'string: %s' % base) + built = signature_method.build_signature(oauth_request, consumer, token) + + def _check_timestamp(self, timestamp): + """Verify that timestamp is recentish.""" + timestamp = int(timestamp) + now = int(time.time()) + lapsed = now - timestamp + if lapsed > self.timestamp_threshold: + raise OAuthError('Expired timestamp: given %d and now %s has a ' + 'greater difference than threshold %d' % + (timestamp, now, self.timestamp_threshold)) + + def _check_nonce(self, consumer, token, nonce): + """Verify that the nonce is uniqueish.""" + nonce = self.data_store.lookup_nonce(consumer, token, nonce) + if nonce: + raise OAuthError('Nonce already used: %s' % str(nonce)) + + +class OAuthClient(object): + """OAuthClient is a worker to attempt to execute a request.""" + consumer = None + token = None + + def __init__(self, oauth_consumer, oauth_token): + self.consumer = oauth_consumer + self.token = oauth_token + + def get_consumer(self): + return self.consumer + + def get_token(self): + return self.token + + def fetch_request_token(self, oauth_request): + """-> OAuthToken.""" + raise NotImplementedError + + def fetch_access_token(self, oauth_request): + """-> OAuthToken.""" + raise NotImplementedError + + def access_resource(self, oauth_request): + """-> Some protected resource.""" + raise NotImplementedError + + +class OAuthDataStore(object): + """A database abstraction used to lookup consumers and tokens.""" + + def lookup_consumer(self, key): + """-> OAuthConsumer.""" + raise NotImplementedError + + def lookup_token(self, oauth_consumer, token_type, token_token): + """-> OAuthToken.""" + raise NotImplementedError + + def lookup_nonce(self, oauth_consumer, oauth_token, nonce): + """-> OAuthToken.""" + raise NotImplementedError + + def fetch_request_token(self, oauth_consumer, oauth_callback): + """-> OAuthToken.""" + raise NotImplementedError + + def fetch_access_token(self, oauth_consumer, oauth_token, oauth_verifier): + """-> OAuthToken.""" + raise NotImplementedError + + def authorize_request_token(self, oauth_token, user): + """-> OAuthToken.""" + raise NotImplementedError + + +class OAuthSignatureMethod(object): + """A strategy class that implements a signature method.""" + def get_name(self): + """-> str.""" + raise NotImplementedError + + def build_signature_base_string(self, oauth_request, oauth_consumer, oauth_token): + """-> str key, str raw.""" + raise NotImplementedError + + def build_signature(self, oauth_request, oauth_consumer, oauth_token): + """-> str.""" + raise NotImplementedError + + def check_signature(self, oauth_request, consumer, token, signature): + built = self.build_signature(oauth_request, consumer, token) + return built == signature + + +class OAuthSignatureMethod_HMAC_SHA1(OAuthSignatureMethod): + + def get_name(self): + return 'HMAC-SHA1' + + def build_signature_base_string(self, oauth_request, consumer, token): + sig = ( + escape(oauth_request.get_normalized_http_method()), + escape(oauth_request.get_normalized_http_url()), + escape(oauth_request.get_normalized_parameters()), + ) + + key = '%s&' % escape(consumer.secret) + if token: + key += escape(token.secret) + raw = '&'.join(sig) + return key, raw + + def build_signature(self, oauth_request, consumer, token): + """Builds the base signature string.""" + key, raw = self.build_signature_base_string(oauth_request, consumer, + token) + + # HMAC object. + try: + import hashlib # 2.5 + hashed = hmac.new(key, raw, hashlib.sha1) + except: + import sha # Deprecated + hashed = hmac.new(key, raw, sha) + + # Calculate the digest base 64. + return binascii.b2a_base64(hashed.digest())[:-1] + + +class OAuthSignatureMethod_PLAINTEXT(OAuthSignatureMethod): + + def get_name(self): + return 'PLAINTEXT' + + def build_signature_base_string(self, oauth_request, consumer, token): + """Concatenates the consumer key and secret.""" + sig = '%s&' % escape(consumer.secret) + if token: + sig = sig + escape(token.secret) + return sig, sig + + def build_signature(self, oauth_request, consumer, token): + key, raw = self.build_signature_base_string(oauth_request, consumer, + token) + return key \ No newline at end of file diff --git a/tweepy/__init__.py b/tweepy/__init__.py new file mode 100644 index 0000000..ff0005f --- /dev/null +++ b/tweepy/__init__.py @@ -0,0 +1,27 @@ +# Tweepy +# Copyright 2009-2010 Joshua Roesslein +# See LICENSE for details. + +""" +Tweepy Twitter API library +""" +__version__ = '2.3' +__author__ = 'Joshua Roesslein' +__license__ = 'MIT' + +from tweepy.models import Status, User, DirectMessage, Friendship, SavedSearch, SearchResults, ModelFactory, Category +from tweepy.error import TweepError +from tweepy.api import API +from tweepy.cache import Cache, MemoryCache, FileCache +from tweepy.auth import OAuthHandler +from tweepy.streaming import Stream, StreamListener +from tweepy.cursor import Cursor + +# Global, unauthenticated instance of API +api = API() + +def debug(enable=True, level=1): + + import httplib + httplib.HTTPConnection.debuglevel = level + diff --git a/tweepy/api.py b/tweepy/api.py new file mode 100644 index 0000000..1d53079 --- /dev/null +++ b/tweepy/api.py @@ -0,0 +1,740 @@ +# Tweepy +# Copyright 2009-2010 Joshua Roesslein +# See LICENSE for details. + +import os +import mimetypes + +from tweepy.binder import bind_api +from tweepy.error import TweepError +from tweepy.parsers import ModelParser +from tweepy.utils import list_to_csv + + +class API(object): + """Twitter API""" + + def __init__(self, auth_handler=None, + host='api.twitter.com', search_host='search.twitter.com', + cache=None, secure=True, api_root='/1.1', search_root='', + retry_count=0, retry_delay=0, retry_errors=None, timeout=60, + parser=None, compression=False): + self.auth = auth_handler + self.host = host + self.search_host = search_host + self.api_root = api_root + self.search_root = search_root + self.cache = cache + self.secure = secure + self.compression = compression + self.retry_count = retry_count + self.retry_delay = retry_delay + self.retry_errors = retry_errors + self.timeout = timeout + self.parser = parser or ModelParser() + + """ statuses/home_timeline """ + home_timeline = bind_api( + path = '/statuses/home_timeline.json', + payload_type = 'status', payload_list = True, + allowed_param = ['since_id', 'max_id', 'count'], + require_auth = True + ) + + """ statuses/user_timeline """ + user_timeline = bind_api( + path = '/statuses/user_timeline.json', + payload_type = 'status', payload_list = True, + allowed_param = ['id', 'user_id', 'screen_name', 'since_id', + 'max_id', 'count', 'include_rts'] + ) + + """ statuses/mentions """ + mentions_timeline = bind_api( + path = '/statuses/mentions_timeline.json', + payload_type = 'status', payload_list = True, + allowed_param = ['since_id', 'max_id', 'count'], + require_auth = True + ) + + """/related_results/show/:id.format""" + related_results = bind_api( + path = '/related_results/show/{id}.json', + payload_type = 'relation', payload_list = True, + allowed_param = ['id'], + require_auth = False + ) + + """ statuses/retweets_of_me """ + retweets_of_me = bind_api( + path = '/statuses/retweets_of_me.json', + payload_type = 'status', payload_list = True, + allowed_param = ['since_id', 'max_id', 'count'], + require_auth = True + ) + + """ statuses/show """ + get_status = bind_api( + path = '/statuses/show.json', + payload_type = 'status', + allowed_param = ['id'] + ) + + """ statuses/update """ + update_status = bind_api( + path = '/statuses/update.json', + method = 'POST', + payload_type = 'status', + allowed_param = ['status', 'in_reply_to_status_id', 'lat', 'long', 'source', 'place_id'], + require_auth = True + ) + + """ statuses/update_with_media """ + def update_with_media(self, filename, *args, **kwargs): + headers, post_data = API._pack_image(filename, 3072, form_field='media[]') + kwargs.update({'headers': headers, 'post_data': post_data}) + + return bind_api( + path='/statuses/update_with_media.json', + method = 'POST', + payload_type='status', + allowed_param = [ + 'status', 'possibly_sensitive', 'in_reply_to_status_id', 'lat', 'long', + 'place_id', 'display_coordinates' + ], + require_auth=True + )(self, *args, **kwargs) + + """ statuses/destroy """ + destroy_status = bind_api( + path = '/statuses/destroy/{id}.json', + method = 'POST', + payload_type = 'status', + allowed_param = ['id'], + require_auth = True + ) + + """ statuses/retweet """ + retweet = bind_api( + path = '/statuses/retweet/{id}.json', + method = 'POST', + payload_type = 'status', + allowed_param = ['id'], + require_auth = True + ) + + """ statuses/retweets """ + retweets = bind_api( + path = '/statuses/retweets/{id}.json', + payload_type = 'status', payload_list = True, + allowed_param = ['id', 'count'], + require_auth = True + ) + + retweeters = bind_api( + path = '/statuses/retweeters/ids.json', + payload_type = 'ids', + allowed_param = ['id', 'cursor', 'stringify_ids'] + ) + + """ users/show """ + get_user = bind_api( + path = '/users/show.json', + payload_type = 'user', + allowed_param = ['id', 'user_id', 'screen_name'] + ) + + ''' statuses/oembed ''' + get_oembed = bind_api( + path = '/statuses/oembed.json', + payload_type = 'json', + allowed_param = ['id', 'url', 'maxwidth', 'hide_media', 'omit_script', 'align', 'related', 'lang'] + ) + + """ Perform bulk look up of users from user ID or screenname """ + def lookup_users(self, user_ids=None, screen_names=None): + return self._lookup_users(list_to_csv(user_ids), list_to_csv(screen_names)) + + _lookup_users = bind_api( + path = '/users/lookup.json', + payload_type = 'user', payload_list = True, + allowed_param = ['user_id', 'screen_name'], + ) + + """ Get the authenticated user """ + def me(self): + return self.get_user(screen_name=self.auth.get_username()) + + """ users/search """ + search_users = bind_api( + path = '/users/search.json', + payload_type = 'user', payload_list = True, + require_auth = True, + allowed_param = ['q', 'count', 'page'] + ) + + """ users/suggestions/:slug """ + suggested_users = bind_api( + path = '/users/suggestions/{slug}.json', + payload_type = 'user', payload_list = True, + require_auth = True, + allowed_param = ['slug', 'lang'] + ) + + """ users/suggestions """ + suggested_categories = bind_api( + path = '/users/suggestions.json', + payload_type = 'category', payload_list = True, + allowed_param = ['lang'], + require_auth = True + ) + + """ users/suggestions/:slug/members """ + suggested_users_tweets = bind_api( + path = '/users/suggestions/{slug}/members.json', + payload_type = 'status', payload_list = True, + allowed_param = ['slug'], + require_auth = True + ) + + """ direct_messages """ + direct_messages = bind_api( + path = '/direct_messages.json', + payload_type = 'direct_message', payload_list = True, + allowed_param = ['since_id', 'max_id', 'count'], + require_auth = True + ) + + """ direct_messages/show """ + get_direct_message = bind_api( + path = '/direct_messages/show/{id}.json', + payload_type = 'direct_message', + allowed_param = ['id'], + require_auth = True + ) + + """ direct_messages/sent """ + sent_direct_messages = bind_api( + path = '/direct_messages/sent.json', + payload_type = 'direct_message', payload_list = True, + allowed_param = ['since_id', 'max_id', 'count', 'page'], + require_auth = True + ) + + """ direct_messages/new """ + send_direct_message = bind_api( + path = '/direct_messages/new.json', + method = 'POST', + payload_type = 'direct_message', + allowed_param = ['user', 'screen_name', 'user_id', 'text'], + require_auth = True + ) + + """ direct_messages/destroy """ + destroy_direct_message = bind_api( + path = '/direct_messages/destroy.json', + method = 'DELETE', + payload_type = 'direct_message', + allowed_param = ['id'], + require_auth = True + ) + + """ friendships/create """ + create_friendship = bind_api( + path = '/friendships/create.json', + method = 'POST', + payload_type = 'user', + allowed_param = ['id', 'user_id', 'screen_name', 'follow'], + require_auth = True + ) + + """ friendships/destroy """ + destroy_friendship = bind_api( + path = '/friendships/destroy.json', + method = 'DELETE', + payload_type = 'user', + allowed_param = ['id', 'user_id', 'screen_name'], + require_auth = True + ) + + """ friendships/show """ + show_friendship = bind_api( + path = '/friendships/show.json', + payload_type = 'friendship', + allowed_param = ['source_id', 'source_screen_name', + 'target_id', 'target_screen_name'] + ) + + """ Perform bulk look up of friendships from user ID or screenname """ + def lookup_friendships(self, user_ids=None, screen_names=None): + return self._lookup_friendships(list_to_csv(user_ids), list_to_csv(screen_names)) + + _lookup_friendships = bind_api( + path = '/friendships/lookup.json', + payload_type = 'relationship', payload_list = True, + allowed_param = ['user_id', 'screen_name'], + require_auth = True + ) + + + """ friends/ids """ + friends_ids = bind_api( + path = '/friends/ids.json', + payload_type = 'ids', + allowed_param = ['id', 'user_id', 'screen_name', 'cursor'] + ) + + """ friends/list """ + friends = bind_api( + path = '/friends/list.json', + payload_type = 'user', payload_list = True, + allowed_param = ['id', 'user_id', 'screen_name', 'cursor'] + ) + + """ friendships/incoming """ + friendships_incoming = bind_api( + path = '/friendships/incoming.json', + payload_type = 'ids', + allowed_param = ['cursor'] + ) + + """ friendships/outgoing""" + friendships_outgoing = bind_api( + path = '/friendships/outgoing.json', + payload_type = 'ids', + allowed_param = ['cursor'] + ) + + """ followers/ids """ + followers_ids = bind_api( + path = '/followers/ids.json', + payload_type = 'ids', + allowed_param = ['id', 'user_id', 'screen_name', 'cursor'] + ) + + """ followers/list """ + followers = bind_api( + path = '/followers/list.json', + payload_type = 'user', payload_list = True, + allowed_param = ['id', 'user_id', 'screen_name', 'cursor', 'count', + 'skip_status', 'include_user_entities'] + ) + + """ account/verify_credentials """ + def verify_credentials(self, **kargs): + try: + return bind_api( + path = '/account/verify_credentials.json', + payload_type = 'user', + require_auth = True, + allowed_param = ['include_entities', 'skip_status'], + )(self, **kargs) + except TweepError as e: + if e.response and e.response.status == 401: + return False + raise + + """ account/rate_limit_status """ + rate_limit_status = bind_api( + path = '/application/rate_limit_status.json', + payload_type = 'json', + allowed_param = ['resources'], + use_cache = False + ) + + """ account/update_delivery_device """ + set_delivery_device = bind_api( + path = '/account/update_delivery_device.json', + method = 'POST', + allowed_param = ['device'], + payload_type = 'user', + require_auth = True + ) + + """ account/update_profile_colors """ + update_profile_colors = bind_api( + path = '/account/update_profile_colors.json', + method = 'POST', + payload_type = 'user', + allowed_param = ['profile_background_color', 'profile_text_color', + 'profile_link_color', 'profile_sidebar_fill_color', + 'profile_sidebar_border_color'], + require_auth = True + ) + + """ account/update_profile_image """ + def update_profile_image(self, filename): + headers, post_data = API._pack_image(filename, 700) + return bind_api( + path = '/account/update_profile_image.json', + method = 'POST', + payload_type = 'user', + require_auth = True + )(self, post_data=post_data, headers=headers) + + """ account/update_profile_background_image """ + def update_profile_background_image(self, filename, *args, **kargs): + headers, post_data = API._pack_image(filename, 800) + bind_api( + path = '/account/update_profile_background_image.json', + method = 'POST', + payload_type = 'user', + allowed_param = ['tile'], + require_auth = True + )(self, post_data=post_data, headers=headers) + + """ account/update_profile_banner """ + def update_profile_banner(self, filename, *args, **kargs): + headers, post_data = API._pack_image(filename, 700, form_field="banner") + bind_api( + path = '/account/update_profile_banner.json', + method = 'POST', + allowed_param = ['width', 'height', 'offset_left', 'offset_right'], + require_auth = True + )(self, post_data=post_data, headers=headers) + + + """ account/update_profile """ + update_profile = bind_api( + path = '/account/update_profile.json', + method = 'POST', + payload_type = 'user', + allowed_param = ['name', 'url', 'location', 'description'], + require_auth = True + ) + + """ favorites """ + favorites = bind_api( + path = '/favorites/list.json', + payload_type = 'status', payload_list = True, + allowed_param = ['screen_name', 'user_id', 'max_id', 'count', 'since_id', 'max_id'] + ) + + """ favorites/create """ + create_favorite = bind_api( + path = '/favorites/create.json', + method = 'POST', + payload_type = 'status', + allowed_param = ['id'], + require_auth = True + ) + + """ favorites/destroy """ + destroy_favorite = bind_api( + path = '/favorites/destroy.json', + method = 'POST', + payload_type = 'status', + allowed_param = ['id'], + require_auth = True + ) + + """ blocks/create """ + create_block = bind_api( + path = '/blocks/create.json', + method = 'POST', + payload_type = 'user', + allowed_param = ['id', 'user_id', 'screen_name'], + require_auth = True + ) + + """ blocks/destroy """ + destroy_block = bind_api( + path = '/blocks/destroy.json', + method = 'DELETE', + payload_type = 'user', + allowed_param = ['id', 'user_id', 'screen_name'], + require_auth = True + ) + + """ blocks/blocking """ + blocks = bind_api( + path = '/blocks/list.json', + payload_type = 'user', payload_list = True, + allowed_param = ['cursor'], + require_auth = True + ) + + """ blocks/blocking/ids """ + blocks_ids = bind_api( + path = '/blocks/ids.json', + payload_type = 'json', + require_auth = True + ) + + """ report_spam """ + report_spam = bind_api( + path = '/users/report_spam.json', + method = 'POST', + payload_type = 'user', + allowed_param = ['user_id', 'screen_name'], + require_auth = True + ) + + """ saved_searches """ + saved_searches = bind_api( + path = '/saved_searches/list.json', + payload_type = 'saved_search', payload_list = True, + require_auth = True + ) + + """ saved_searches/show """ + get_saved_search = bind_api( + path = '/saved_searches/show/{id}.json', + payload_type = 'saved_search', + allowed_param = ['id'], + require_auth = True + ) + + """ saved_searches/create """ + create_saved_search = bind_api( + path = '/saved_searches/create.json', + method = 'POST', + payload_type = 'saved_search', + allowed_param = ['query'], + require_auth = True + ) + + """ saved_searches/destroy """ + destroy_saved_search = bind_api( + path = '/saved_searches/destroy/{id}.json', + method = 'POST', + payload_type = 'saved_search', + allowed_param = ['id'], + require_auth = True + ) + + create_list = bind_api( + path = '/lists/create.json', + method = 'POST', + payload_type = 'list', + allowed_param = ['name', 'mode', 'description'], + require_auth = True + ) + + destroy_list = bind_api( + path = '/lists/destroy.json', + method = 'POST', + payload_type = 'list', + allowed_param = ['owner_screen_name', 'owner_id', 'list_id', 'slug'], + require_auth = True + ) + + update_list = bind_api( + path = '/lists/update.json', + method = 'POST', + payload_type = 'list', + allowed_param = ['list_id', 'slug', 'name', 'mode', 'description', 'owner_screen_name', 'owner_id'], + require_auth = True + ) + + lists_all = bind_api( + path = '/lists/list.json', + payload_type = 'list', payload_list = True, + allowed_param = ['screen_name', 'user_id'], + require_auth = True + ) + + lists_memberships = bind_api( + path = '/lists/memberships.json', + payload_type = 'list', payload_list = True, + allowed_param = ['screen_name', 'user_id', 'filter_to_owned_lists', 'cursor'], + require_auth = True + ) + + lists_subscriptions = bind_api( + path = '/lists/subscriptions.json', + payload_type = 'list', payload_list = True, + allowed_param = ['screen_name', 'user_id', 'cursor'], + require_auth = True + ) + + list_timeline = bind_api( + path = '/lists/statuses.json', + payload_type = 'status', payload_list = True, + allowed_param = ['owner_screen_name', 'slug', 'owner_id', 'list_id', 'since_id', 'max_id', 'count', 'include_rts'] + ) + + get_list = bind_api( + path = '/lists/show.json', + payload_type = 'list', + allowed_param = ['owner_screen_name', 'owner_id', 'slug', 'list_id'] + ) + + add_list_member = bind_api( + path = '/lists/members/create.json', + method = 'POST', + payload_type = 'list', + allowed_param = ['screen_name', 'user_id', 'owner_screen_name', 'owner_id', 'slug', 'list_id'], + require_auth = True + ) + + remove_list_member = bind_api( + path = '/lists/members/destroy.json', + method = 'POST', + payload_type = 'list', + allowed_param = ['screen_name', 'user_id', 'owner_screen_name', 'owner_id', 'slug', 'list_id'], + require_auth = True + ) + + list_members = bind_api( + path = '/lists/members.json', + payload_type = 'user', payload_list = True, + allowed_param = ['owner_screen_name', 'slug', 'list_id', 'owner_id', 'cursor'] + ) + + show_list_member = bind_api( + path = '/lists/members/show.json', + payload_type = 'user', + allowed_param = ['list_id', 'slug', 'user_id', 'screen_name', 'owner_screen_name', 'owner_id'] + ) + + subscribe_list = bind_api( + path = '/lists/subscribers/create.json', + method = 'POST', + payload_type = 'list', + allowed_param = ['owner_screen_name', 'slug', 'owner_id', 'list_id'], + require_auth = True + ) + + unsubscribe_list = bind_api( + path = '/lists/subscribers/destroy.json', + method = 'POST', + payload_type = 'list', + allowed_param = ['owner_screen_name', 'slug', 'owner_id', 'list_id'], + require_auth = True + ) + + list_subscribers = bind_api( + path = '/lists/subscribers.json', + payload_type = 'user', payload_list = True, + allowed_param = ['owner_screen_name', 'slug', 'owner_id', 'list_id', 'cursor'] + ) + + show_list_subscriber = bind_api( + path = '/lists/subscribers/show.json', + payload_type = 'user', + allowed_param = ['owner_screen_name', 'slug', 'screen_name', 'owner_id', 'list_id', 'user_id'] + ) + + """ trends/available """ + trends_available = bind_api( + path = '/trends/available.json', + payload_type = 'json' + ) + + trends_place = bind_api( + path = '/trends/place.json', + payload_type = 'json', + allowed_param = ['id', 'exclude'] + ) + + trends_closest = bind_api( + path = '/trends/closest.json', + payload_type = 'json', + allowed_param = ['lat', 'long'] + ) + + """ search """ + search = bind_api( + path = '/search/tweets.json', + payload_type = 'search_results', + allowed_param = ['q', 'lang', 'locale', 'since_id', 'geocode', 'max_id', 'since', 'until', 'result_type', 'count', 'include_entities', 'from', 'to', 'source'] + ) + + """ trends/daily """ + trends_daily = bind_api( + path = '/trends/daily.json', + payload_type = 'json', + allowed_param = ['date', 'exclude'] + ) + + """ trends/weekly """ + trends_weekly = bind_api( + path = '/trends/weekly.json', + payload_type = 'json', + allowed_param = ['date', 'exclude'] + ) + + """ geo/reverse_geocode """ + reverse_geocode = bind_api( + path = '/geo/reverse_geocode.json', + payload_type = 'place', payload_list = True, + allowed_param = ['lat', 'long', 'accuracy', 'granularity', 'max_results'] + ) + + """ geo/id """ + geo_id = bind_api( + path = '/geo/id/{id}.json', + payload_type = 'place', + allowed_param = ['id'] + ) + + """ geo/search """ + geo_search = bind_api( + path = '/geo/search.json', + payload_type = 'place', payload_list = True, + allowed_param = ['lat', 'long', 'query', 'ip', 'granularity', 'accuracy', 'max_results', 'contained_within'] + ) + + """ geo/similar_places """ + geo_similar_places = bind_api( + path = '/geo/similar_places.json', + payload_type = 'place', payload_list = True, + allowed_param = ['lat', 'long', 'name', 'contained_within'] + ) + + """ help/languages.json """ + supported_languages = bind_api( + path = '/help/languages.json', + payload_type = 'json', + require_auth = True + ) + + """ help/configuration """ + configuration = bind_api( + path = '/help/configuration.json', + payload_type = 'json', + require_auth = True + ) + + """ Internal use only """ + @staticmethod + def _pack_image(filename, max_size, form_field="image"): + """Pack image from file into multipart-formdata post body""" + # image must be less than 700kb in size + try: + if os.path.getsize(filename) > (max_size * 1024): + raise TweepError('File is too big, must be less than 700kb.') + except os.error: + raise TweepError('Unable to access file') + + # image must be gif, jpeg, or png + file_type = mimetypes.guess_type(filename) + if file_type is None: + raise TweepError('Could not determine file type') + file_type = file_type[0] + if file_type not in ['image/gif', 'image/jpeg', 'image/png']: + raise TweepError('Invalid file type for image: %s' % file_type) + + # build the mulitpart-formdata body + fp = open(filename, 'rb') + BOUNDARY = 'Tw3ePy' + body = [] + body.append('--' + BOUNDARY) + body.append('Content-Disposition: form-data; name="%s"; filename="%s"' % (form_field, filename)) + body.append('Content-Type: %s' % file_type) + body.append('') + body.append(fp.read()) + body.append('--' + BOUNDARY + '--') + body.append('') + fp.close() + body = '\r\n'.join(body) + + # build headers + headers = { + 'Content-Type': 'multipart/form-data; boundary=Tw3ePy', + 'Content-Length': str(len(body)) + } + + return headers, body + diff --git a/tweepy/auth.py b/tweepy/auth.py new file mode 100644 index 0000000..86c4430 --- /dev/null +++ b/tweepy/auth.py @@ -0,0 +1,156 @@ +# Tweepy +# Copyright 2009-2010 Joshua Roesslein +# See LICENSE for details. + +from urllib2 import Request, urlopen +import base64 + +from tweepy import oauth +from tweepy.error import TweepError +from tweepy.api import API + + +class AuthHandler(object): + + def apply_auth(self, url, method, headers, parameters): + """Apply authentication headers to request""" + raise NotImplementedError + + def get_username(self): + """Return the username of the authenticated user""" + raise NotImplementedError + + +class OAuthHandler(AuthHandler): + """OAuth authentication handler""" + + OAUTH_HOST = 'api.twitter.com' + OAUTH_ROOT = '/oauth/' + + def __init__(self, consumer_key, consumer_secret, callback=None, secure=True): + if type(consumer_key) == unicode: + consumer_key = bytes(consumer_key) + + if type(consumer_secret) == unicode: + consumer_secret = bytes(consumer_secret) + + self._consumer = oauth.OAuthConsumer(consumer_key, consumer_secret) + self._sigmethod = oauth.OAuthSignatureMethod_HMAC_SHA1() + self.request_token = None + self.access_token = None + self.callback = callback + self.username = None + self.secure = secure + + def _get_oauth_url(self, endpoint, secure=True): + if self.secure or secure: + prefix = 'https://' + else: + prefix = 'http://' + + return prefix + self.OAUTH_HOST + self.OAUTH_ROOT + endpoint + + def apply_auth(self, url, method, headers, parameters): + request = oauth.OAuthRequest.from_consumer_and_token( + self._consumer, http_url=url, http_method=method, + token=self.access_token, parameters=parameters + ) + request.sign_request(self._sigmethod, self._consumer, self.access_token) + headers.update(request.to_header()) + + def _get_request_token(self): + try: + url = self._get_oauth_url('request_token') + request = oauth.OAuthRequest.from_consumer_and_token( + self._consumer, http_url=url, callback=self.callback + ) + request.sign_request(self._sigmethod, self._consumer, None) + resp = urlopen(Request(url, headers=request.to_header())) + return oauth.OAuthToken.from_string(resp.read()) + except Exception as e: + raise TweepError(e) + + def set_request_token(self, key, secret): + self.request_token = oauth.OAuthToken(key, secret) + + def set_access_token(self, key, secret): + self.access_token = oauth.OAuthToken(key, secret) + + def get_authorization_url(self, signin_with_twitter=False): + """Get the authorization URL to redirect the user""" + try: + # get the request token + self.request_token = self._get_request_token() + + # build auth request and return as url + if signin_with_twitter: + url = self._get_oauth_url('authenticate') + else: + url = self._get_oauth_url('authorize') + request = oauth.OAuthRequest.from_token_and_callback( + token=self.request_token, http_url=url + ) + + return request.to_url() + except Exception as e: + raise TweepError(e) + + def get_access_token(self, verifier=None): + """ + After user has authorized the request token, get access token + with user supplied verifier. + """ + try: + url = self._get_oauth_url('access_token') + + # build request + request = oauth.OAuthRequest.from_consumer_and_token( + self._consumer, + token=self.request_token, http_url=url, + verifier=str(verifier) + ) + request.sign_request(self._sigmethod, self._consumer, self.request_token) + + # send request + resp = urlopen(Request(url, headers=request.to_header())) + self.access_token = oauth.OAuthToken.from_string(resp.read()) + return self.access_token + except Exception as e: + raise TweepError(e) + + def get_xauth_access_token(self, username, password): + """ + Get an access token from an username and password combination. + In order to get this working you need to create an app at + http://twitter.com/apps, after that send a mail to api@twitter.com + and request activation of xAuth for it. + """ + try: + url = self._get_oauth_url('access_token', secure=True) # must use HTTPS + request = oauth.OAuthRequest.from_consumer_and_token( + oauth_consumer=self._consumer, + http_method='POST', http_url=url, + parameters = { + 'x_auth_mode': 'client_auth', + 'x_auth_username': username, + 'x_auth_password': password + } + ) + request.sign_request(self._sigmethod, self._consumer, None) + + resp = urlopen(Request(url, data=request.to_postdata())) + self.access_token = oauth.OAuthToken.from_string(resp.read()) + return self.access_token + except Exception as e: + raise TweepError(e) + + def get_username(self): + if self.username is None: + api = API(self) + user = api.verify_credentials() + if user: + self.username = user.screen_name + else: + raise TweepError("Unable to get username, invalid oauth token!") + return self.username + diff --git a/tweepy/binder.py b/tweepy/binder.py new file mode 100644 index 0000000..1f8c619 --- /dev/null +++ b/tweepy/binder.py @@ -0,0 +1,213 @@ +# Tweepy +# Copyright 2009-2010 Joshua Roesslein +# See LICENSE for details. + +import httplib +import urllib +import time +import re +from StringIO import StringIO +import gzip + +from tweepy.error import TweepError +from tweepy.utils import convert_to_utf8_str +from tweepy.models import Model + +re_path_template = re.compile('{\w+}') + + +def bind_api(**config): + + class APIMethod(object): + + path = config['path'] + payload_type = config.get('payload_type', None) + payload_list = config.get('payload_list', False) + allowed_param = config.get('allowed_param', []) + method = config.get('method', 'GET') + require_auth = config.get('require_auth', False) + search_api = config.get('search_api', False) + use_cache = config.get('use_cache', True) + + def __init__(self, api, args, kargs): + # If authentication is required and no credentials + # are provided, throw an error. + if self.require_auth and not api.auth: + raise TweepError('Authentication required!') + + self.api = api + self.post_data = kargs.pop('post_data', None) + self.retry_count = kargs.pop('retry_count', api.retry_count) + self.retry_delay = kargs.pop('retry_delay', api.retry_delay) + self.retry_errors = kargs.pop('retry_errors', api.retry_errors) + self.headers = kargs.pop('headers', {}) + self.build_parameters(args, kargs) + + # Pick correct URL root to use + if self.search_api: + self.api_root = api.search_root + else: + self.api_root = api.api_root + + # Perform any path variable substitution + self.build_path() + + if api.secure: + self.scheme = 'https://' + else: + self.scheme = 'http://' + + if self.search_api: + self.host = api.search_host + else: + self.host = api.host + + # Manually set Host header to fix an issue in python 2.5 + # or older where Host is set including the 443 port. + # This causes Twitter to issue 301 redirect. + # See Issue https://github.com/tweepy/tweepy/issues/12 + self.headers['Host'] = self.host + + def build_parameters(self, args, kargs): + self.parameters = {} + for idx, arg in enumerate(args): + if arg is None: + continue + + try: + self.parameters[self.allowed_param[idx]] = convert_to_utf8_str(arg) + except IndexError: + raise TweepError('Too many parameters supplied!') + + for k, arg in kargs.items(): + if arg is None: + continue + if k in self.parameters: + raise TweepError('Multiple values for parameter %s supplied!' % k) + + self.parameters[k] = convert_to_utf8_str(arg) + + def build_path(self): + for variable in re_path_template.findall(self.path): + name = variable.strip('{}') + + if name == 'user' and 'user' not in self.parameters and self.api.auth: + # No 'user' parameter provided, fetch it from Auth instead. + value = self.api.auth.get_username() + else: + try: + value = urllib.quote(self.parameters[name]) + except KeyError: + raise TweepError('No parameter value found for path variable: %s' % name) + del self.parameters[name] + + self.path = self.path.replace(variable, value) + + def execute(self): + self.api.cached_result = False + + # Build the request URL + url = self.api_root + self.path + if len(self.parameters): + url = '%s?%s' % (url, urllib.urlencode(self.parameters)) + + # Query the cache if one is available + # and this request uses a GET method. + if self.use_cache and self.api.cache and self.method == 'GET': + cache_result = self.api.cache.get(url) + # if cache result found and not expired, return it + if cache_result: + # must restore api reference + if isinstance(cache_result, list): + for result in cache_result: + if isinstance(result, Model): + result._api = self.api + else: + if isinstance(cache_result, Model): + cache_result._api = self.api + self.api.cached_result = True + return cache_result + + # Continue attempting request until successful + # or maximum number of retries is reached. + retries_performed = 0 + while retries_performed < self.retry_count + 1: + # Open connection + if self.api.secure: + conn = httplib.HTTPSConnection(self.host, timeout=self.api.timeout) + else: + conn = httplib.HTTPConnection(self.host, timeout=self.api.timeout) + + # Apply authentication + if self.api.auth: + self.api.auth.apply_auth( + self.scheme + self.host + url, + self.method, self.headers, self.parameters + ) + + # Request compression if configured + if self.api.compression: + self.headers['Accept-encoding'] = 'gzip' + + # Execute request + try: + conn.request(self.method, url, headers=self.headers, body=self.post_data) + resp = conn.getresponse() + except Exception as e: + raise TweepError('Failed to send request: %s' % e) + + # Exit request loop if non-retry error code + if self.retry_errors: + if resp.status not in self.retry_errors: break + else: + if resp.status == 200: break + + # Sleep before retrying request again + time.sleep(self.retry_delay) + retries_performed += 1 + + # If an error was returned, throw an exception + self.api.last_response = resp + if resp.status and not 200 <= resp.status < 300: + try: + error_msg = self.api.parser.parse_error(resp.read()) + except Exception: + error_msg = "Twitter error response: status code = %s" % resp.status + raise TweepError(error_msg, resp) + + # Parse the response payload + body = resp.read() + if resp.getheader('Content-Encoding', '') == 'gzip': + try: + zipper = gzip.GzipFile(fileobj=StringIO(body)) + body = zipper.read() + except Exception as e: + raise TweepError('Failed to decompress data: %s' % e) + result = self.api.parser.parse(self, body) + + conn.close() + + # Store result into cache if one is available. + if self.use_cache and self.api.cache and self.method == 'GET' and result: + self.api.cache.store(url, result) + + return result + + + def _call(api, *args, **kargs): + + method = APIMethod(api, args, kargs) + return method.execute() + + + # Set pagination mode + if 'cursor' in APIMethod.allowed_param: + _call.pagination_mode = 'cursor' + elif 'max_id' in APIMethod.allowed_param and \ + 'since_id' in APIMethod.allowed_param: + _call.pagination_mode = 'id' + elif 'page' in APIMethod.allowed_param: + _call.pagination_mode = 'page' + + return _call + diff --git a/tweepy/cache.py b/tweepy/cache.py new file mode 100644 index 0000000..a50a349 --- /dev/null +++ b/tweepy/cache.py @@ -0,0 +1,424 @@ +# Tweepy +# Copyright 2009-2010 Joshua Roesslein +# See LICENSE for details. + +import time +import datetime +import threading +import os + +try: + import cPickle as pickle +except ImportError: + import pickle + +try: + import hashlib +except ImportError: + # python 2.4 + import md5 as hashlib + +try: + import fcntl +except ImportError: + # Probably on a windows system + # TODO: use win32file + pass + + +class Cache(object): + """Cache interface""" + + def __init__(self, timeout=60): + """Initialize the cache + timeout: number of seconds to keep a cached entry + """ + self.timeout = timeout + + def store(self, key, value): + """Add new record to cache + key: entry key + value: data of entry + """ + raise NotImplementedError + + def get(self, key, timeout=None): + """Get cached entry if exists and not expired + key: which entry to get + timeout: override timeout with this value [optional] + """ + raise NotImplementedError + + def count(self): + """Get count of entries currently stored in cache""" + raise NotImplementedError + + def cleanup(self): + """Delete any expired entries in cache.""" + raise NotImplementedError + + def flush(self): + """Delete all cached entries""" + raise NotImplementedError + + +class MemoryCache(Cache): + """In-memory cache""" + + def __init__(self, timeout=60): + Cache.__init__(self, timeout) + self._entries = {} + self.lock = threading.Lock() + + def __getstate__(self): + # pickle + return {'entries': self._entries, 'timeout': self.timeout} + + def __setstate__(self, state): + # unpickle + self.lock = threading.Lock() + self._entries = state['entries'] + self.timeout = state['timeout'] + + def _is_expired(self, entry, timeout): + return timeout > 0 and (time.time() - entry[0]) >= timeout + + def store(self, key, value): + self.lock.acquire() + self._entries[key] = (time.time(), value) + self.lock.release() + + def get(self, key, timeout=None): + self.lock.acquire() + try: + # check to see if we have this key + entry = self._entries.get(key) + if not entry: + # no hit, return nothing + return None + + # use provided timeout in arguments if provided + # otherwise use the one provided during init. + if timeout is None: + timeout = self.timeout + + # make sure entry is not expired + if self._is_expired(entry, timeout): + # entry expired, delete and return nothing + del self._entries[key] + return None + + # entry found and not expired, return it + return entry[1] + finally: + self.lock.release() + + def count(self): + return len(self._entries) + + def cleanup(self): + self.lock.acquire() + try: + for k, v in self._entries.items(): + if self._is_expired(v, self.timeout): + del self._entries[k] + finally: + self.lock.release() + + def flush(self): + self.lock.acquire() + self._entries.clear() + self.lock.release() + + +class FileCache(Cache): + """File-based cache""" + + # locks used to make cache thread-safe + cache_locks = {} + + def __init__(self, cache_dir, timeout=60): + Cache.__init__(self, timeout) + if os.path.exists(cache_dir) is False: + os.mkdir(cache_dir) + self.cache_dir = cache_dir + if cache_dir in FileCache.cache_locks: + self.lock = FileCache.cache_locks[cache_dir] + else: + self.lock = threading.Lock() + FileCache.cache_locks[cache_dir] = self.lock + + if os.name == 'posix': + self._lock_file = self._lock_file_posix + self._unlock_file = self._unlock_file_posix + elif os.name == 'nt': + self._lock_file = self._lock_file_win32 + self._unlock_file = self._unlock_file_win32 + else: + print('Warning! FileCache locking not supported on this system!') + self._lock_file = self._lock_file_dummy + self._unlock_file = self._unlock_file_dummy + + def _get_path(self, key): + md5 = hashlib.md5() + md5.update(key) + return os.path.join(self.cache_dir, md5.hexdigest()) + + def _lock_file_dummy(self, path, exclusive=True): + return None + + def _unlock_file_dummy(self, lock): + return + + def _lock_file_posix(self, path, exclusive=True): + lock_path = path + '.lock' + if exclusive is True: + f_lock = open(lock_path, 'w') + fcntl.lockf(f_lock, fcntl.LOCK_EX) + else: + f_lock = open(lock_path, 'r') + fcntl.lockf(f_lock, fcntl.LOCK_SH) + if os.path.exists(lock_path) is False: + f_lock.close() + return None + return f_lock + + def _unlock_file_posix(self, lock): + lock.close() + + def _lock_file_win32(self, path, exclusive=True): + # TODO: implement + return None + + def _unlock_file_win32(self, lock): + # TODO: implement + return + + def _delete_file(self, path): + os.remove(path) + if os.path.exists(path + '.lock'): + os.remove(path + '.lock') + + def store(self, key, value): + path = self._get_path(key) + self.lock.acquire() + try: + # acquire lock and open file + f_lock = self._lock_file(path) + datafile = open(path, 'wb') + + # write data + pickle.dump((time.time(), value), datafile) + + # close and unlock file + datafile.close() + self._unlock_file(f_lock) + finally: + self.lock.release() + + def get(self, key, timeout=None): + return self._get(self._get_path(key), timeout) + + def _get(self, path, timeout): + if os.path.exists(path) is False: + # no record + return None + self.lock.acquire() + try: + # acquire lock and open + f_lock = self._lock_file(path, False) + datafile = open(path, 'rb') + + # read pickled object + created_time, value = pickle.load(datafile) + datafile.close() + + # check if value is expired + if timeout is None: + timeout = self.timeout + if timeout > 0 and (time.time() - created_time) >= timeout: + # expired! delete from cache + value = None + self._delete_file(path) + + # unlock and return result + self._unlock_file(f_lock) + return value + finally: + self.lock.release() + + def count(self): + c = 0 + for entry in os.listdir(self.cache_dir): + if entry.endswith('.lock'): + continue + c += 1 + return c + + def cleanup(self): + for entry in os.listdir(self.cache_dir): + if entry.endswith('.lock'): + continue + self._get(os.path.join(self.cache_dir, entry), None) + + def flush(self): + for entry in os.listdir(self.cache_dir): + if entry.endswith('.lock'): + continue + self._delete_file(os.path.join(self.cache_dir, entry)) + +class MemCacheCache(Cache): + """Cache interface""" + + def __init__(self, client, timeout=60): + """Initialize the cache + client: The memcache client + timeout: number of seconds to keep a cached entry + """ + self.client = client + self.timeout = timeout + + def store(self, key, value): + """Add new record to cache + key: entry key + value: data of entry + """ + self.client.set(key, value, time=self.timeout) + + def get(self, key, timeout=None): + """Get cached entry if exists and not expired + key: which entry to get + timeout: override timeout with this value [optional]. DOES NOT WORK HERE + """ + return self.client.get(key) + + def count(self): + """Get count of entries currently stored in cache. RETURN 0""" + raise NotImplementedError + + def cleanup(self): + """Delete any expired entries in cache. NO-OP""" + raise NotImplementedError + + def flush(self): + """Delete all cached entries. NO-OP""" + raise NotImplementedError + +class RedisCache(Cache): + '''Cache running in a redis server''' + + def __init__(self, client, timeout=60, keys_container = 'tweepy:keys', pre_identifier = 'tweepy:'): + Cache.__init__(self, timeout) + self.client = client + self.keys_container = keys_container + self.pre_identifier = pre_identifier + + def _is_expired(self, entry, timeout): + # Returns true if the entry has expired + return timeout > 0 and (time.time() - entry[0]) >= timeout + + def store(self, key, value): + '''Store the key, value pair in our redis server''' + # Prepend tweepy to our key, this makes it easier to identify tweepy keys in our redis server + key = self.pre_identifier + key + # Get a pipe (to execute several redis commands in one step) + pipe = self.client.pipeline() + # Set our values in a redis hash (similar to python dict) + pipe.set(key, pickle.dumps((time.time(), value))) + # Set the expiration + pipe.expire(key, self.timeout) + # Add the key to a set containing all the keys + pipe.sadd(self.keys_container, key) + # Execute the instructions in the redis server + pipe.execute() + + def get(self, key, timeout=None): + '''Given a key, returns an element from the redis table''' + key = self.pre_identifier + key + # Check to see if we have this key + unpickled_entry = self.client.get(key) + if not unpickled_entry: + # No hit, return nothing + return None + + entry = pickle.loads(unpickled_entry) + # Use provided timeout in arguments if provided + # otherwise use the one provided during init. + if timeout is None: + timeout = self.timeout + + # Make sure entry is not expired + if self._is_expired(entry, timeout): + # entry expired, delete and return nothing + self.delete_entry(key) + return None + # entry found and not expired, return it + return entry[1] + + def count(self): + '''Note: This is not very efficient, since it retreives all the keys from the redis + server to know how many keys we have''' + return len(self.client.smembers(self.keys_container)) + + def delete_entry(self, key): + '''Delete an object from the redis table''' + pipe = self.client.pipeline() + pipe.srem(self.keys_container, key) + pipe.delete(key) + pipe.execute() + + def cleanup(self): + '''Cleanup all the expired keys''' + keys = self.client.smembers(self.keys_container) + for key in keys: + entry = self.client.get(key) + if entry: + entry = pickle.loads(entry) + if self._is_expired(entry, self.timeout): + self.delete_entry(key) + + def flush(self): + '''Delete all entries from the cache''' + keys = self.client.smembers(self.keys_container) + for key in keys: + self.delete_entry(key) + + +class MongodbCache(Cache): + """A simple pickle-based MongoDB cache sytem.""" + + def __init__(self, db, timeout=3600, collection='tweepy_cache'): + """Should receive a "database" cursor from pymongo.""" + Cache.__init__(self, timeout) + self.timeout = timeout + self.col = db[collection] + self.col.create_index('created', expireAfterSeconds=timeout) + + def store(self, key, value): + from bson.binary import Binary + + now = datetime.datetime.utcnow() + blob = Binary(pickle.dumps(value)) + + self.col.insert({'created': now, '_id': key, 'value': blob}) + + def get(self, key, timeout=None): + if timeout: + raise NotImplementedError + obj = self.col.find_one({'_id': key}) + if obj: + return pickle.loads(obj['value']) + + def count(self): + return self.col.find({}).count() + + def delete_entry(self, key): + return self.col.remove({'_id': key}) + + def cleanup(self): + """MongoDB will automatically clear expired keys.""" + pass + + def flush(self): + self.col.drop() + self.col.create_index('created', expireAfterSeconds=self.timeout) diff --git a/tweepy/cursor.py b/tweepy/cursor.py new file mode 100644 index 0000000..4c06f17 --- /dev/null +++ b/tweepy/cursor.py @@ -0,0 +1,171 @@ +# Tweepy +# Copyright 2009-2010 Joshua Roesslein +# See LICENSE for details. + +from tweepy.error import TweepError + +class Cursor(object): + """Pagination helper class""" + + def __init__(self, method, *args, **kargs): + if hasattr(method, 'pagination_mode'): + if method.pagination_mode == 'cursor': + self.iterator = CursorIterator(method, args, kargs) + elif method.pagination_mode == 'id': + self.iterator = IdIterator(method, args, kargs) + elif method.pagination_mode == 'page': + self.iterator = PageIterator(method, args, kargs) + else: + raise TweepError('Invalid pagination mode.') + else: + raise TweepError('This method does not perform pagination') + + def pages(self, limit=0): + """Return iterator for pages""" + if limit > 0: + self.iterator.limit = limit + return self.iterator + + def items(self, limit=0): + """Return iterator for items in each page""" + i = ItemIterator(self.iterator) + i.limit = limit + return i + +class BaseIterator(object): + + def __init__(self, method, args, kargs): + self.method = method + self.args = args + self.kargs = kargs + self.limit = 0 + + def next(self): + raise NotImplementedError + + def prev(self): + raise NotImplementedError + + def __iter__(self): + return self + +class CursorIterator(BaseIterator): + + def __init__(self, method, args, kargs): + BaseIterator.__init__(self, method, args, kargs) + start_cursor = kargs.pop('cursor', None) + self.next_cursor = start_cursor or -1 + self.prev_cursor = start_cursor or 0 + self.count = 0 + + def next(self): + if self.next_cursor == 0 or (self.limit and self.count == self.limit): + raise StopIteration + data, cursors = self.method( + cursor=self.next_cursor, *self.args, **self.kargs + ) + self.prev_cursor, self.next_cursor = cursors + if len(data) == 0: + raise StopIteration + self.count += 1 + return data + + def prev(self): + if self.prev_cursor == 0: + raise TweepError('Can not page back more, at first page') + data, self.next_cursor, self.prev_cursor = self.method( + cursor=self.prev_cursor, *self.args, **self.kargs + ) + self.count -= 1 + return data + +class IdIterator(BaseIterator): + + def __init__(self, method, args, kargs): + BaseIterator.__init__(self, method, args, kargs) + self.max_id = kargs.get('max_id') + self.since_id = kargs.get('since_id') + self.count = 0 + + def next(self): + """Fetch a set of items with IDs less than current set.""" + if self.limit and self.limit == self.count: + raise StopIteration + + # max_id is inclusive so decrement by one + # to avoid requesting duplicate items. + max_id = self.since_id - 1 if self.max_id else None + data = self.method(max_id = max_id, *self.args, **self.kargs) + if len(data) == 0: + raise StopIteration + self.max_id = data.max_id + self.since_id = data.since_id + self.count += 1 + return data + + def prev(self): + """Fetch a set of items with IDs greater than current set.""" + if self.limit and self.limit == self.count: + raise StopIteration + + since_id = self.max_id + data = self.method(since_id = since_id, *self.args, **self.kargs) + if len(data) == 0: + raise StopIteration + self.max_id = data.max_id + self.since_id = data.since_id + self.count += 1 + return data + +class PageIterator(BaseIterator): + + def __init__(self, method, args, kargs): + BaseIterator.__init__(self, method, args, kargs) + self.current_page = 0 + + def next(self): + self.current_page += 1 + items = self.method(page=self.current_page, *self.args, **self.kargs) + if len(items) == 0 or (self.limit > 0 and self.current_page > self.limit): + raise StopIteration + return items + + def prev(self): + if (self.current_page == 1): + raise TweepError('Can not page back more, at first page') + self.current_page -= 1 + return self.method(page=self.current_page, *self.args, **self.kargs) + +class ItemIterator(BaseIterator): + + def __init__(self, page_iterator): + self.page_iterator = page_iterator + self.limit = 0 + self.current_page = None + self.page_index = -1 + self.count = 0 + + def next(self): + if self.limit > 0 and self.count == self.limit: + raise StopIteration + if self.current_page is None or self.page_index == len(self.current_page) - 1: + # Reached end of current page, get the next page... + self.current_page = self.page_iterator.next() + self.page_index = -1 + self.page_index += 1 + self.count += 1 + return self.current_page[self.page_index] + + def prev(self): + if self.current_page is None: + raise TweepError('Can not go back more, at first page') + if self.page_index == 0: + # At the beginning of the current page, move to next... + self.current_page = self.page_iterator.prev() + self.page_index = len(self.current_page) + if self.page_index == 0: + raise TweepError('No more items') + self.page_index -= 1 + self.count -= 1 + return self.current_page[self.page_index] + diff --git a/tweepy/error.py b/tweepy/error.py new file mode 100644 index 0000000..753e2fe --- /dev/null +++ b/tweepy/error.py @@ -0,0 +1,15 @@ +# Tweepy +# Copyright 2009-2010 Joshua Roesslein +# See LICENSE for details. + +class TweepError(Exception): + """Tweepy exception""" + + def __init__(self, reason, response=None): + self.reason = unicode(reason) + self.response = response + Exception.__init__(self, reason) + + def __str__(self): + return self.reason + diff --git a/tweepy/models.py b/tweepy/models.py new file mode 100644 index 0000000..1338ab4 --- /dev/null +++ b/tweepy/models.py @@ -0,0 +1,436 @@ +# Tweepy +# Copyright 2009-2010 Joshua Roesslein +# See LICENSE for details. + +from tweepy.error import TweepError +from tweepy.utils import parse_datetime, parse_html_value, parse_a_href + + +class ResultSet(list): + """A list like object that holds results from a Twitter API query.""" + def __init__(self, max_id=None, since_id=None): + super(ResultSet, self).__init__() + self._max_id = max_id + self._since_id = since_id + + @property + def max_id(self): + if self._max_id: + return self._max_id + ids = self.ids() + return max(ids) if ids else None + + @property + def since_id(self): + if self._since_id: + return self._since_id + ids = self.ids() + return min(ids) if ids else None + + def ids(self): + return [item.id for item in self if hasattr(item, 'id')] + +class Model(object): + + def __init__(self, api=None): + self._api = api + + def __getstate__(self): + # pickle + pickle = dict(self.__dict__) + try: + del pickle['_api'] # do not pickle the API reference + except KeyError: + pass + return pickle + + @classmethod + def parse(cls, api, json): + """Parse a JSON object into a model instance.""" + raise NotImplementedError + + @classmethod + def parse_list(cls, api, json_list): + """Parse a list of JSON objects into a result set of model instances.""" + results = ResultSet() + for obj in json_list: + if obj: + results.append(cls.parse(api, obj)) + return results + + def __repr__(self): + state = ['%s=%s' % (k, repr(v)) for (k,v) in vars(self).items()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(state)) + + +class Status(Model): + + @classmethod + def parse(cls, api, json): + status = cls(api) + for k, v in json.items(): + if k == 'user': + user_model = getattr(api.parser.model_factory, 'user') if api else User + user = user_model.parse(api, v) + setattr(status, 'author', user) + setattr(status, 'user', user) # DEPRECIATED + elif k == 'created_at': + setattr(status, k, parse_datetime(v)) + elif k == 'source': + if '<' in v: + setattr(status, k, parse_html_value(v)) + setattr(status, 'source_url', parse_a_href(v)) + else: + setattr(status, k, v) + setattr(status, 'source_url', None) + elif k == 'retweeted_status': + setattr(status, k, Status.parse(api, v)) + elif k == 'place': + if v is not None: + setattr(status, k, Place.parse(api, v)) + else: + setattr(status, k, None) + else: + setattr(status, k, v) + return status + + def destroy(self): + return self._api.destroy_status(self.id) + + def retweet(self): + return self._api.retweet(self.id) + + def retweets(self): + return self._api.retweets(self.id) + + def favorite(self): + return self._api.create_favorite(self.id) + + +class User(Model): + + @classmethod + def parse(cls, api, json): + user = cls(api) + for k, v in json.items(): + if k == 'created_at': + setattr(user, k, parse_datetime(v)) + elif k == 'status': + setattr(user, k, Status.parse(api, v)) + elif k == 'following': + # twitter sets this to null if it is false + if v is True: + setattr(user, k, True) + else: + setattr(user, k, False) + else: + setattr(user, k, v) + return user + + @classmethod + def parse_list(cls, api, json_list): + if isinstance(json_list, list): + item_list = json_list + else: + item_list = json_list['users'] + + results = ResultSet() + for obj in item_list: + results.append(cls.parse(api, obj)) + return results + + def timeline(self, **kargs): + return self._api.user_timeline(user_id=self.id, **kargs) + + def friends(self, **kargs): + return self._api.friends(user_id=self.id, **kargs) + + def followers(self, **kargs): + return self._api.followers(user_id=self.id, **kargs) + + def follow(self): + self._api.create_friendship(user_id=self.id) + self.following = True + + def unfollow(self): + self._api.destroy_friendship(user_id=self.id) + self.following = False + + def lists_memberships(self, *args, **kargs): + return self._api.lists_memberships(user=self.screen_name, *args, **kargs) + + def lists_subscriptions(self, *args, **kargs): + return self._api.lists_subscriptions(user=self.screen_name, *args, **kargs) + + def lists(self, *args, **kargs): + return self._api.lists_all(user=self.screen_name, *args, **kargs) + + def followers_ids(self, *args, **kargs): + return self._api.followers_ids(user_id=self.id, *args, **kargs) + + +class DirectMessage(Model): + + @classmethod + def parse(cls, api, json): + dm = cls(api) + for k, v in json.items(): + if k == 'sender' or k == 'recipient': + setattr(dm, k, User.parse(api, v)) + elif k == 'created_at': + setattr(dm, k, parse_datetime(v)) + else: + setattr(dm, k, v) + return dm + + def destroy(self): + return self._api.destroy_direct_message(self.id) + + +class Friendship(Model): + + @classmethod + def parse(cls, api, json): + relationship = json['relationship'] + + # parse source + source = cls(api) + for k, v in relationship['source'].items(): + setattr(source, k, v) + + # parse target + target = cls(api) + for k, v in relationship['target'].items(): + setattr(target, k, v) + + return source, target + + +class Category(Model): + + @classmethod + def parse(cls, api, json): + category = cls(api) + for k, v in json.items(): + setattr(category, k, v) + return category + + +class SavedSearch(Model): + + @classmethod + def parse(cls, api, json): + ss = cls(api) + for k, v in json.items(): + if k == 'created_at': + setattr(ss, k, parse_datetime(v)) + else: + setattr(ss, k, v) + return ss + + def destroy(self): + return self._api.destroy_saved_search(self.id) + + +class SearchResults(ResultSet): + + @classmethod + def parse(cls, api, json): + metadata = json['search_metadata'] + results = SearchResults(metadata.get('max_id'), metadata.get('since_id')) + results.refresh_url = metadata.get('refresh_url') + results.completed_in = metadata.get('completed_in') + results.query = metadata.get('query') + results.count = metadata.get('count') + results.next_results = metadata.get('next_results') + + for status in json['statuses']: + results.append(Status.parse(api, status)) + return results + + +class List(Model): + + @classmethod + def parse(cls, api, json): + lst = List(api) + for k,v in json.items(): + if k == 'user': + setattr(lst, k, User.parse(api, v)) + elif k == 'created_at': + setattr(lst, k, parse_datetime(v)) + else: + setattr(lst, k, v) + return lst + + @classmethod + def parse_list(cls, api, json_list, result_set=None): + results = ResultSet() + if isinstance(json_list, dict): + json_list = json_list['lists'] + for obj in json_list: + results.append(cls.parse(api, obj)) + return results + + def update(self, **kargs): + return self._api.update_list(self.slug, **kargs) + + def destroy(self): + return self._api.destroy_list(self.slug) + + def timeline(self, **kargs): + return self._api.list_timeline(self.user.screen_name, self.slug, **kargs) + + def add_member(self, id): + return self._api.add_list_member(self.slug, id) + + def remove_member(self, id): + return self._api.remove_list_member(self.slug, id) + + def members(self, **kargs): + return self._api.list_members(self.user.screen_name, self.slug, **kargs) + + def is_member(self, id): + return self._api.is_list_member(self.user.screen_name, self.slug, id) + + def subscribe(self): + return self._api.subscribe_list(self.user.screen_name, self.slug) + + def unsubscribe(self): + return self._api.unsubscribe_list(self.user.screen_name, self.slug) + + def subscribers(self, **kargs): + return self._api.list_subscribers(self.user.screen_name, self.slug, **kargs) + + def is_subscribed(self, id): + return self._api.is_subscribed_list(self.user.screen_name, self.slug, id) + +class Relation(Model): + @classmethod + def parse(cls, api, json): + result = cls(api) + for k,v in json.items(): + if k == 'value' and json['kind'] in ['Tweet', 'LookedupStatus']: + setattr(result, k, Status.parse(api, v)) + elif k == 'results': + setattr(result, k, Relation.parse_list(api, v)) + else: + setattr(result, k, v) + return result + +class Relationship(Model): + @classmethod + def parse(cls, api, json): + result = cls(api) + for k,v in json.items(): + if k == 'connections': + setattr(result, 'is_following', 'following' in v) + setattr(result, 'is_followed_by', 'followed_by' in v) + else: + setattr(result, k, v) + return result + +class JSONModel(Model): + + @classmethod + def parse(cls, api, json): + return json + + +class IDModel(Model): + + @classmethod + def parse(cls, api, json): + if isinstance(json, list): + return json + else: + return json['ids'] + + +class BoundingBox(Model): + + @classmethod + def parse(cls, api, json): + result = cls(api) + if json is not None: + for k, v in json.items(): + setattr(result, k, v) + return result + + def origin(self): + """ + Return longitude, latitude of southwest (bottom, left) corner of + bounding box, as a tuple. + + This assumes that bounding box is always a rectangle, which + appears to be the case at present. + """ + return tuple(self.coordinates[0][0]) + + def corner(self): + """ + Return longitude, latitude of northeast (top, right) corner of + bounding box, as a tuple. + + This assumes that bounding box is always a rectangle, which + appears to be the case at present. + """ + return tuple(self.coordinates[0][2]) + + +class Place(Model): + + @classmethod + def parse(cls, api, json): + place = cls(api) + for k, v in json.items(): + if k == 'bounding_box': + # bounding_box value may be null (None.) + # Example: "United States" (id=96683cc9126741d1) + if v is not None: + t = BoundingBox.parse(api, v) + else: + t = v + setattr(place, k, t) + elif k == 'contained_within': + # contained_within is a list of Places. + setattr(place, k, Place.parse_list(api, v)) + else: + setattr(place, k, v) + return place + + @classmethod + def parse_list(cls, api, json_list): + if isinstance(json_list, list): + item_list = json_list + else: + item_list = json_list['result']['places'] + + results = ResultSet() + for obj in item_list: + results.append(cls.parse(api, obj)) + return results + +class ModelFactory(object): + """ + Used by parsers for creating instances + of models. You may subclass this factory + to add your own extended models. + """ + + status = Status + user = User + direct_message = DirectMessage + friendship = Friendship + saved_search = SavedSearch + search_results = SearchResults + category = Category + list = List + relation = Relation + relationship = Relationship + + json = JSONModel + ids = IDModel + place = Place + bounding_box = BoundingBox + diff --git a/tweepy/oauth.py b/tweepy/oauth.py new file mode 100644 index 0000000..286de18 --- /dev/null +++ b/tweepy/oauth.py @@ -0,0 +1,655 @@ +""" +The MIT License + +Copyright (c) 2007 Leah Culver + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +""" + +import cgi +import urllib +import time +import random +import urlparse +import hmac +import binascii + + +VERSION = '1.0' # Hi Blaine! +HTTP_METHOD = 'GET' +SIGNATURE_METHOD = 'PLAINTEXT' + + +class OAuthError(RuntimeError): + """Generic exception class.""" + def __init__(self, message='OAuth error occured.'): + self.message = message + +def build_authenticate_header(realm=''): + """Optional WWW-Authenticate header (401 error)""" + return {'WWW-Authenticate': 'OAuth realm="%s"' % realm} + +def escape(s): + """Escape a URL including any /.""" + return urllib.quote(s, safe='~') + +def _utf8_str(s): + """Convert unicode to utf-8.""" + if isinstance(s, unicode): + return s.encode("utf-8") + else: + return str(s) + +def generate_timestamp(): + """Get seconds since epoch (UTC).""" + return int(time.time()) + +def generate_nonce(length=8): + """Generate pseudorandom number.""" + return ''.join([str(random.randint(0, 9)) for i in range(length)]) + +def generate_verifier(length=8): + """Generate pseudorandom number.""" + return ''.join([str(random.randint(0, 9)) for i in range(length)]) + + +class OAuthConsumer(object): + """Consumer of OAuth authentication. + + OAuthConsumer is a data type that represents the identity of the Consumer + via its shared secret with the Service Provider. + + """ + key = None + secret = None + + def __init__(self, key, secret): + self.key = key + self.secret = secret + + +class OAuthToken(object): + """OAuthToken is a data type that represents an End User via either an access + or request token. + + key -- the token + secret -- the token secret + + """ + key = None + secret = None + callback = None + callback_confirmed = None + verifier = None + + def __init__(self, key, secret): + self.key = key + self.secret = secret + + def set_callback(self, callback): + self.callback = callback + self.callback_confirmed = 'true' + + def set_verifier(self, verifier=None): + if verifier is not None: + self.verifier = verifier + else: + self.verifier = generate_verifier() + + def get_callback_url(self): + if self.callback and self.verifier: + # Append the oauth_verifier. + parts = urlparse.urlparse(self.callback) + scheme, netloc, path, params, query, fragment = parts[:6] + if query: + query = '%s&oauth_verifier=%s' % (query, self.verifier) + else: + query = 'oauth_verifier=%s' % self.verifier + return urlparse.urlunparse((scheme, netloc, path, params, + query, fragment)) + return self.callback + + def to_string(self): + data = { + 'oauth_token': self.key, + 'oauth_token_secret': self.secret, + } + if self.callback_confirmed is not None: + data['oauth_callback_confirmed'] = self.callback_confirmed + return urllib.urlencode(data) + + def from_string(s): + """ Returns a token from something like: + oauth_token_secret=xxx&oauth_token=xxx + """ + params = cgi.parse_qs(s, keep_blank_values=False) + key = params['oauth_token'][0] + secret = params['oauth_token_secret'][0] + token = OAuthToken(key, secret) + try: + token.callback_confirmed = params['oauth_callback_confirmed'][0] + except KeyError: + pass # 1.0, no callback confirmed. + return token + from_string = staticmethod(from_string) + + def __str__(self): + return self.to_string() + + +class OAuthRequest(object): + """OAuthRequest represents the request and can be serialized. + + OAuth parameters: + - oauth_consumer_key + - oauth_token + - oauth_signature_method + - oauth_signature + - oauth_timestamp + - oauth_nonce + - oauth_version + - oauth_verifier + ... any additional parameters, as defined by the Service Provider. + """ + parameters = None # OAuth parameters. + http_method = HTTP_METHOD + http_url = None + version = VERSION + + def __init__(self, http_method=HTTP_METHOD, http_url=None, parameters=None): + self.http_method = http_method + self.http_url = http_url + self.parameters = parameters or {} + + def set_parameter(self, parameter, value): + self.parameters[parameter] = value + + def get_parameter(self, parameter): + try: + return self.parameters[parameter] + except: + raise OAuthError('Parameter not found: %s' % parameter) + + def _get_timestamp_nonce(self): + return self.get_parameter('oauth_timestamp'), self.get_parameter( + 'oauth_nonce') + + def get_nonoauth_parameters(self): + """Get any non-OAuth parameters.""" + parameters = {} + for k, v in self.parameters.iteritems(): + # Ignore oauth parameters. + if k.find('oauth_') < 0: + parameters[k] = v + return parameters + + def to_header(self, realm=''): + """Serialize as a header for an HTTPAuth request.""" + auth_header = 'OAuth realm="%s"' % realm + # Add the oauth parameters. + if self.parameters: + for k, v in self.parameters.iteritems(): + if k[:6] == 'oauth_': + auth_header += ', %s="%s"' % (k, escape(str(v))) + return {'Authorization': auth_header} + + def to_postdata(self): + """Serialize as post data for a POST request.""" + return '&'.join(['%s=%s' % (escape(str(k)), escape(str(v))) \ + for k, v in self.parameters.iteritems()]) + + def to_url(self): + """Serialize as a URL for a GET request.""" + return '%s?%s' % (self.get_normalized_http_url(), self.to_postdata()) + + def get_normalized_parameters(self): + """Return a string that contains the parameters that must be signed.""" + params = self.parameters + try: + # Exclude the signature if it exists. + del params['oauth_signature'] + except: + pass + # Escape key values before sorting. + key_values = [(escape(_utf8_str(k)), escape(_utf8_str(v))) \ + for k,v in params.items()] + # Sort lexicographically, first after key, then after value. + key_values.sort() + # Combine key value pairs into a string. + return '&'.join(['%s=%s' % (k, v) for k, v in key_values]) + + def get_normalized_http_method(self): + """Uppercases the http method.""" + return self.http_method.upper() + + def get_normalized_http_url(self): + """Parses the URL and rebuilds it to be scheme://host/path.""" + parts = urlparse.urlparse(self.http_url) + scheme, netloc, path = parts[:3] + # Exclude default port numbers. + if scheme == 'http' and netloc[-3:] == ':80': + netloc = netloc[:-3] + elif scheme == 'https' and netloc[-4:] == ':443': + netloc = netloc[:-4] + return '%s://%s%s' % (scheme, netloc, path) + + def sign_request(self, signature_method, consumer, token): + """Set the signature parameter to the result of build_signature.""" + # Set the signature method. + self.set_parameter('oauth_signature_method', + signature_method.get_name()) + # Set the signature. + self.set_parameter('oauth_signature', + self.build_signature(signature_method, consumer, token)) + + def build_signature(self, signature_method, consumer, token): + """Calls the build signature method within the signature method.""" + return signature_method.build_signature(self, consumer, token) + + def from_request(http_method, http_url, headers=None, parameters=None, + query_string=None): + """Combines multiple parameter sources.""" + if parameters is None: + parameters = {} + + # Headers + if headers and 'Authorization' in headers: + auth_header = headers['Authorization'] + # Check that the authorization header is OAuth. + if auth_header[:6] == 'OAuth ': + auth_header = auth_header[6:] + try: + # Get the parameters from the header. + header_params = OAuthRequest._split_header(auth_header) + parameters.update(header_params) + except: + raise OAuthError('Unable to parse OAuth parameters from ' + 'Authorization header.') + + # GET or POST query string. + if query_string: + query_params = OAuthRequest._split_url_string(query_string) + parameters.update(query_params) + + # URL parameters. + param_str = urlparse.urlparse(http_url)[4] # query + url_params = OAuthRequest._split_url_string(param_str) + parameters.update(url_params) + + if parameters: + return OAuthRequest(http_method, http_url, parameters) + + return None + from_request = staticmethod(from_request) + + def from_consumer_and_token(oauth_consumer, token=None, + callback=None, verifier=None, http_method=HTTP_METHOD, + http_url=None, parameters=None): + if not parameters: + parameters = {} + + defaults = { + 'oauth_consumer_key': oauth_consumer.key, + 'oauth_timestamp': generate_timestamp(), + 'oauth_nonce': generate_nonce(), + 'oauth_version': OAuthRequest.version, + } + + defaults.update(parameters) + parameters = defaults + + if token: + parameters['oauth_token'] = token.key + if token.callback: + parameters['oauth_callback'] = token.callback + # 1.0a support for verifier. + if verifier: + parameters['oauth_verifier'] = verifier + elif callback: + # 1.0a support for callback in the request token request. + parameters['oauth_callback'] = callback + + return OAuthRequest(http_method, http_url, parameters) + from_consumer_and_token = staticmethod(from_consumer_and_token) + + def from_token_and_callback(token, callback=None, http_method=HTTP_METHOD, + http_url=None, parameters=None): + if not parameters: + parameters = {} + + parameters['oauth_token'] = token.key + + if callback: + parameters['oauth_callback'] = callback + + return OAuthRequest(http_method, http_url, parameters) + from_token_and_callback = staticmethod(from_token_and_callback) + + def _split_header(header): + """Turn Authorization: header into parameters.""" + params = {} + parts = header.split(',') + for param in parts: + # Ignore realm parameter. + if param.find('realm') > -1: + continue + # Remove whitespace. + param = param.strip() + # Split key-value. + param_parts = param.split('=', 1) + # Remove quotes and unescape the value. + params[param_parts[0]] = urllib.unquote(param_parts[1].strip('\"')) + return params + _split_header = staticmethod(_split_header) + + def _split_url_string(param_str): + """Turn URL string into parameters.""" + parameters = cgi.parse_qs(param_str, keep_blank_values=False) + for k, v in parameters.iteritems(): + parameters[k] = urllib.unquote(v[0]) + return parameters + _split_url_string = staticmethod(_split_url_string) + +class OAuthServer(object): + """A worker to check the validity of a request against a data store.""" + timestamp_threshold = 300 # In seconds, five minutes. + version = VERSION + signature_methods = None + data_store = None + + def __init__(self, data_store=None, signature_methods=None): + self.data_store = data_store + self.signature_methods = signature_methods or {} + + def set_data_store(self, data_store): + self.data_store = data_store + + def get_data_store(self): + return self.data_store + + def add_signature_method(self, signature_method): + self.signature_methods[signature_method.get_name()] = signature_method + return self.signature_methods + + def fetch_request_token(self, oauth_request): + """Processes a request_token request and returns the + request token on success. + """ + try: + # Get the request token for authorization. + token = self._get_token(oauth_request, 'request') + except OAuthError: + # No token required for the initial token request. + version = self._get_version(oauth_request) + consumer = self._get_consumer(oauth_request) + try: + callback = self.get_callback(oauth_request) + except OAuthError: + callback = None # 1.0, no callback specified. + self._check_signature(oauth_request, consumer, None) + # Fetch a new token. + token = self.data_store.fetch_request_token(consumer, callback) + return token + + def fetch_access_token(self, oauth_request): + """Processes an access_token request and returns the + access token on success. + """ + version = self._get_version(oauth_request) + consumer = self._get_consumer(oauth_request) + try: + verifier = self._get_verifier(oauth_request) + except OAuthError: + verifier = None + # Get the request token. + token = self._get_token(oauth_request, 'request') + self._check_signature(oauth_request, consumer, token) + new_token = self.data_store.fetch_access_token(consumer, token, verifier) + return new_token + + def verify_request(self, oauth_request): + """Verifies an api call and checks all the parameters.""" + # -> consumer and token + version = self._get_version(oauth_request) + consumer = self._get_consumer(oauth_request) + # Get the access token. + token = self._get_token(oauth_request, 'access') + self._check_signature(oauth_request, consumer, token) + parameters = oauth_request.get_nonoauth_parameters() + return consumer, token, parameters + + def authorize_token(self, token, user): + """Authorize a request token.""" + return self.data_store.authorize_request_token(token, user) + + def get_callback(self, oauth_request): + """Get the callback URL.""" + return oauth_request.get_parameter('oauth_callback') + + def build_authenticate_header(self, realm=''): + """Optional support for the authenticate header.""" + return {'WWW-Authenticate': 'OAuth realm="%s"' % realm} + + def _get_version(self, oauth_request): + """Verify the correct version request for this server.""" + try: + version = oauth_request.get_parameter('oauth_version') + except: + version = VERSION + if version and version != self.version: + raise OAuthError('OAuth version %s not supported.' % str(version)) + return version + + def _get_signature_method(self, oauth_request): + """Figure out the signature with some defaults.""" + try: + signature_method = oauth_request.get_parameter( + 'oauth_signature_method') + except: + signature_method = SIGNATURE_METHOD + try: + # Get the signature method object. + signature_method = self.signature_methods[signature_method] + except: + signature_method_names = ', '.join(self.signature_methods.keys()) + raise OAuthError('Signature method %s not supported try one of the ' + 'following: %s' % (signature_method, signature_method_names)) + + return signature_method + + def _get_consumer(self, oauth_request): + consumer_key = oauth_request.get_parameter('oauth_consumer_key') + consumer = self.data_store.lookup_consumer(consumer_key) + if not consumer: + raise OAuthError('Invalid consumer.') + return consumer + + def _get_token(self, oauth_request, token_type='access'): + """Try to find the token for the provided request token key.""" + token_field = oauth_request.get_parameter('oauth_token') + token = self.data_store.lookup_token(token_type, token_field) + if not token: + raise OAuthError('Invalid %s token: %s' % (token_type, token_field)) + return token + + def _get_verifier(self, oauth_request): + return oauth_request.get_parameter('oauth_verifier') + + def _check_signature(self, oauth_request, consumer, token): + timestamp, nonce = oauth_request._get_timestamp_nonce() + self._check_timestamp(timestamp) + self._check_nonce(consumer, token, nonce) + signature_method = self._get_signature_method(oauth_request) + try: + signature = oauth_request.get_parameter('oauth_signature') + except: + raise OAuthError('Missing signature.') + # Validate the signature. + valid_sig = signature_method.check_signature(oauth_request, consumer, + token, signature) + if not valid_sig: + key, base = signature_method.build_signature_base_string( + oauth_request, consumer, token) + raise OAuthError('Invalid signature. Expected signature base ' + 'string: %s' % base) + built = signature_method.build_signature(oauth_request, consumer, token) + + def _check_timestamp(self, timestamp): + """Verify that timestamp is recentish.""" + timestamp = int(timestamp) + now = int(time.time()) + lapsed = abs(now - timestamp) + if lapsed > self.timestamp_threshold: + raise OAuthError('Expired timestamp: given %d and now %s has a ' + 'greater difference than threshold %d' % + (timestamp, now, self.timestamp_threshold)) + + def _check_nonce(self, consumer, token, nonce): + """Verify that the nonce is uniqueish.""" + nonce = self.data_store.lookup_nonce(consumer, token, nonce) + if nonce: + raise OAuthError('Nonce already used: %s' % str(nonce)) + + +class OAuthClient(object): + """OAuthClient is a worker to attempt to execute a request.""" + consumer = None + token = None + + def __init__(self, oauth_consumer, oauth_token): + self.consumer = oauth_consumer + self.token = oauth_token + + def get_consumer(self): + return self.consumer + + def get_token(self): + return self.token + + def fetch_request_token(self, oauth_request): + """-> OAuthToken.""" + raise NotImplementedError + + def fetch_access_token(self, oauth_request): + """-> OAuthToken.""" + raise NotImplementedError + + def access_resource(self, oauth_request): + """-> Some protected resource.""" + raise NotImplementedError + + +class OAuthDataStore(object): + """A database abstraction used to lookup consumers and tokens.""" + + def lookup_consumer(self, key): + """-> OAuthConsumer.""" + raise NotImplementedError + + def lookup_token(self, oauth_consumer, token_type, token_token): + """-> OAuthToken.""" + raise NotImplementedError + + def lookup_nonce(self, oauth_consumer, oauth_token, nonce): + """-> OAuthToken.""" + raise NotImplementedError + + def fetch_request_token(self, oauth_consumer, oauth_callback): + """-> OAuthToken.""" + raise NotImplementedError + + def fetch_access_token(self, oauth_consumer, oauth_token, oauth_verifier): + """-> OAuthToken.""" + raise NotImplementedError + + def authorize_request_token(self, oauth_token, user): + """-> OAuthToken.""" + raise NotImplementedError + + +class OAuthSignatureMethod(object): + """A strategy class that implements a signature method.""" + def get_name(self): + """-> str.""" + raise NotImplementedError + + def build_signature_base_string(self, oauth_request, oauth_consumer, oauth_token): + """-> str key, str raw.""" + raise NotImplementedError + + def build_signature(self, oauth_request, oauth_consumer, oauth_token): + """-> str.""" + raise NotImplementedError + + def check_signature(self, oauth_request, consumer, token, signature): + built = self.build_signature(oauth_request, consumer, token) + return built == signature + + +class OAuthSignatureMethod_HMAC_SHA1(OAuthSignatureMethod): + + def get_name(self): + return 'HMAC-SHA1' + + def build_signature_base_string(self, oauth_request, consumer, token): + sig = ( + escape(oauth_request.get_normalized_http_method()), + escape(oauth_request.get_normalized_http_url()), + escape(oauth_request.get_normalized_parameters()), + ) + + key = '%s&' % escape(consumer.secret) + if token: + key += escape(token.secret) + raw = '&'.join(sig) + return key, raw + + def build_signature(self, oauth_request, consumer, token): + """Builds the base signature string.""" + key, raw = self.build_signature_base_string(oauth_request, consumer, + token) + + # HMAC object. + try: + import hashlib # 2.5 + hashed = hmac.new(key, raw, hashlib.sha1) + except: + import sha # Deprecated + hashed = hmac.new(key, raw, sha) + + # Calculate the digest base 64. + return binascii.b2a_base64(hashed.digest())[:-1] + + +class OAuthSignatureMethod_PLAINTEXT(OAuthSignatureMethod): + + def get_name(self): + return 'PLAINTEXT' + + def build_signature_base_string(self, oauth_request, consumer, token): + """Concatenates the consumer key and secret.""" + sig = '%s&' % escape(consumer.secret) + if token: + sig = sig + escape(token.secret) + return sig, sig + + def build_signature(self, oauth_request, consumer, token): + key, raw = self.build_signature_base_string(oauth_request, consumer, + token) + return key \ No newline at end of file diff --git a/tweepy/parsers.py b/tweepy/parsers.py new file mode 100644 index 0000000..31e0022 --- /dev/null +++ b/tweepy/parsers.py @@ -0,0 +1,97 @@ +# Tweepy +# Copyright 2009-2010 Joshua Roesslein +# See LICENSE for details. + +from tweepy.models import ModelFactory +from tweepy.utils import import_simplejson +from tweepy.error import TweepError + + +class Parser(object): + + def parse(self, method, payload): + """ + Parse the response payload and return the result. + Returns a tuple that contains the result data and the cursors + (or None if not present). + """ + raise NotImplementedError + + def parse_error(self, payload): + """ + Parse the error message from payload. + If unable to parse the message, throw an exception + and default error message will be used. + """ + raise NotImplementedError + + +class RawParser(Parser): + + def __init__(self): + pass + + def parse(self, method, payload): + return payload + + def parse_error(self, payload): + return payload + + +class JSONParser(Parser): + + payload_format = 'json' + + def __init__(self): + self.json_lib = import_simplejson() + + def parse(self, method, payload): + try: + json = self.json_lib.loads(payload) + except Exception as e: + raise TweepError('Failed to parse JSON payload: %s' % e) + + needsCursors = method.parameters.has_key('cursor') + if needsCursors and isinstance(json, dict) and 'previous_cursor' in json and 'next_cursor' in json: + cursors = json['previous_cursor'], json['next_cursor'] + return json, cursors + else: + return json + + def parse_error(self, payload): + error = self.json_lib.loads(payload) + if error.has_key('error'): + return error['error'] + else: + return error['errors'] + + +class ModelParser(JSONParser): + + def __init__(self, model_factory=None): + JSONParser.__init__(self) + self.model_factory = model_factory or ModelFactory + + def parse(self, method, payload): + try: + if method.payload_type is None: return + model = getattr(self.model_factory, method.payload_type) + except AttributeError: + raise TweepError('No model for this payload type: %s' % method.payload_type) + + json = JSONParser.parse(self, method, payload) + if isinstance(json, tuple): + json, cursors = json + else: + cursors = None + + if method.payload_list: + result = model.parse_list(method.api, json) + else: + result = model.parse(method.api, json) + + if cursors: + return result, cursors + else: + return result + diff --git a/tweepy/streaming.py b/tweepy/streaming.py new file mode 100644 index 0000000..be233ff --- /dev/null +++ b/tweepy/streaming.py @@ -0,0 +1,319 @@ +# Tweepy +# Copyright 2009-2010 Joshua Roesslein +# See LICENSE for details. + +import logging +import httplib +from socket import timeout +from threading import Thread +from time import sleep +import ssl + +from tweepy.models import Status +from tweepy.api import API +from tweepy.error import TweepError + +from tweepy.utils import import_simplejson, urlencode_noplus +json = import_simplejson() + +STREAM_VERSION = '1.1' + + +class StreamListener(object): + + def __init__(self, api=None): + self.api = api or API() + + def on_connect(self): + """Called once connected to streaming server. + + This will be invoked once a successful response + is received from the server. Allows the listener + to perform some work prior to entering the read loop. + """ + pass + + def on_data(self, raw_data): + """Called when raw data is received from connection. + + Override this method if you wish to manually handle + the stream data. Return False to stop stream and close connection. + """ + data = json.loads(raw_data) + + if 'in_reply_to_status_id' in data: + status = Status.parse(self.api, data) + if self.on_status(status) is False: + return False + elif 'delete' in data: + delete = data['delete']['status'] + if self.on_delete(delete['id'], delete['user_id']) is False: + return False + elif 'event' in data: + status = Status.parse(self.api, data) + if self.on_event(status) is False: + return False + elif 'direct_message' in data: + status = Status.parse(self.api, data) + if self.on_direct_message(status) is False: + return False + elif 'limit' in data: + if self.on_limit(data['limit']['track']) is False: + return False + elif 'disconnect' in data: + if self.on_disconnect(data['disconnect']) is False: + return False + else: + logging.error("Unknown message type: " + str(raw_data)) + + def on_status(self, status): + """Called when a new status arrives""" + return + + def on_exception(self, exception): + """Called when an unhandled exception occurs.""" + return + + def on_delete(self, status_id, user_id): + """Called when a delete notice arrives for a status""" + return + + def on_event(self, status): + """Called when a new event arrives""" + return + + def on_direct_message(self, status): + """Called when a new direct message arrives""" + return + + def on_limit(self, track): + """Called when a limitation notice arrvies""" + return + + def on_error(self, status_code): + """Called when a non-200 status code is returned""" + return False + + def on_timeout(self): + """Called when stream connection times out""" + return + + def on_disconnect(self, notice): + """Called when twitter sends a disconnect notice + + Disconnect codes are listed here: + https://dev.twitter.com/docs/streaming-apis/messages#Disconnect_messages_disconnect + """ + return + + +class Stream(object): + + host = 'stream.twitter.com' + + def __init__(self, auth, listener, **options): + self.auth = auth + self.listener = listener + self.running = False + self.timeout = options.get("timeout", 300.0) + self.retry_count = options.get("retry_count") + # values according to https://dev.twitter.com/docs/streaming-apis/connecting#Reconnecting + self.retry_time_start = options.get("retry_time", 5.0) + self.retry_420_start = options.get("retry_420", 60.0) + self.retry_time_cap = options.get("retry_time_cap", 320.0) + self.snooze_time_step = options.get("snooze_time", 0.25) + self.snooze_time_cap = options.get("snooze_time_cap", 16) + self.buffer_size = options.get("buffer_size", 1500) + if options.get("secure", True): + self.scheme = "https" + else: + self.scheme = "http" + + self.api = API() + self.headers = options.get("headers") or {} + self.parameters = None + self.body = None + self.retry_time = self.retry_time_start + self.snooze_time = self.snooze_time_step + + def _run(self): + # Authenticate + url = "%s://%s%s" % (self.scheme, self.host, self.url) + + # Connect and process the stream + error_counter = 0 + conn = None + exception = None + while self.running: + if self.retry_count is not None and error_counter > self.retry_count: + # quit if error count greater than retry count + break + try: + if self.scheme == "http": + conn = httplib.HTTPConnection(self.host, timeout=self.timeout) + else: + conn = httplib.HTTPSConnection(self.host, timeout=self.timeout) + self.auth.apply_auth(url, 'POST', self.headers, self.parameters) + conn.connect() + conn.request('POST', self.url, self.body, headers=self.headers) + resp = conn.getresponse() + if resp.status != 200: + if self.listener.on_error(resp.status) is False: + break + error_counter += 1 + if resp.status == 420: + self.retry_time = max(self.retry_420_start, self.retry_time) + sleep(self.retry_time) + self.retry_time = min(self.retry_time * 2, self.retry_time_cap) + else: + error_counter = 0 + self.retry_time = self.retry_time_start + self.snooze_time = self.snooze_time_step + self.listener.on_connect() + self._read_loop(resp) + except (timeout, ssl.SSLError) as exc: + # If it's not time out treat it like any other exception + if isinstance(exc, ssl.SSLError) and not (exc.args and 'timed out' in str(exc.args[0])): + exception = exc + break + + if self.listener.on_timeout() == False: + break + if self.running is False: + break + conn.close() + sleep(self.snooze_time) + self.snooze_time = min(self.snooze_time + self.snooze_time_step, + self.snooze_time_cap) + except Exception as exception: + # any other exception is fatal, so kill loop + break + + # cleanup + self.running = False + if conn: + conn.close() + + if exception: + # call a handler first so that the exception can be logged. + self.listener.on_exception(exception) + raise + + def _data(self, data): + if self.listener.on_data(data) is False: + self.running = False + + def _read_loop(self, resp): + + while self.running and not resp.isclosed(): + + # Note: keep-alive newlines might be inserted before each length value. + # read until we get a digit... + c = '\n' + while c == '\n' and self.running and not resp.isclosed(): + c = resp.read(1) + delimited_string = c + + # read rest of delimiter length.. + d = '' + while d != '\n' and self.running and not resp.isclosed(): + d = resp.read(1) + delimited_string += d + + # read the next twitter status object + if delimited_string.strip().isdigit(): + next_status_obj = resp.read( int(delimited_string) ) + self._data(next_status_obj) + + if resp.isclosed(): + self.on_closed(resp) + + def _start(self, async): + self.running = True + if async: + Thread(target=self._run).start() + else: + self._run() + + def on_closed(self, resp): + """ Called when the response has been closed by Twitter """ + pass + + def userstream(self, stall_warnings=False, _with=None, replies=None, + track=None, locations=None, async=False, encoding='utf8'): + self.parameters = {'delimited': 'length'} + if self.running: + raise TweepError('Stream object already connected!') + self.url = '/%s/user.json?delimited=length' % STREAM_VERSION + self.host='userstream.twitter.com' + if stall_warnings: + self.parameters['stall_warnings'] = stall_warnings + if _with: + self.parameters['with'] = _with + if replies: + self.parameters['replies'] = replies + if locations and len(locations) > 0: + assert len(locations) % 4 == 0 + self.parameters['locations'] = ','.join(['%.2f' % l for l in locations]) + if track: + encoded_track = [s.encode(encoding) for s in track] + self.parameters['track'] = ','.join(encoded_track) + self.body = urlencode_noplus(self.parameters) + self._start(async) + + def firehose(self, count=None, async=False): + self.parameters = {'delimited': 'length'} + if self.running: + raise TweepError('Stream object already connected!') + self.url = '/%s/statuses/firehose.json?delimited=length' % STREAM_VERSION + if count: + self.url += '&count=%s' % count + self._start(async) + + def retweet(self, async=False): + self.parameters = {'delimited': 'length'} + if self.running: + raise TweepError('Stream object already connected!') + self.url = '/%s/statuses/retweet.json?delimited=length' % STREAM_VERSION + self._start(async) + + def sample(self, count=None, async=False): + self.parameters = {'delimited': 'length'} + if self.running: + raise TweepError('Stream object already connected!') + self.url = '/%s/statuses/sample.json?delimited=length' % STREAM_VERSION + if count: + self.url += '&count=%s' % count + self._start(async) + + def filter(self, follow=None, track=None, async=False, locations=None, + count=None, stall_warnings=False, languages=None, encoding='utf8'): + self.parameters = {} + self.headers['Content-type'] = "application/x-www-form-urlencoded" + if self.running: + raise TweepError('Stream object already connected!') + self.url = '/%s/statuses/filter.json?delimited=length' % STREAM_VERSION + if follow: + encoded_follow = [s.encode(encoding) for s in follow] + self.parameters['follow'] = ','.join(encoded_follow) + if track: + encoded_track = [s.encode(encoding) for s in track] + self.parameters['track'] = ','.join(encoded_track) + if locations and len(locations) > 0: + assert len(locations) % 4 == 0 + self.parameters['locations'] = ','.join(['%.4f' % l for l in locations]) + if count: + self.parameters['count'] = count + if stall_warnings: + self.parameters['stall_warnings'] = stall_warnings + if languages: + self.parameters['language'] = ','.join(map(str, languages)) + self.body = urlencode_noplus(self.parameters) + self.parameters['delimited'] = 'length' + self._start(async) + + def disconnect(self): + if self.running is False: + return + self.running = False + diff --git a/tweepy/utils.py b/tweepy/utils.py new file mode 100644 index 0000000..e5d2a5e --- /dev/null +++ b/tweepy/utils.py @@ -0,0 +1,59 @@ +# Tweepy +# Copyright 2010 Joshua Roesslein +# See LICENSE for details. + +from datetime import datetime +import time +import re +import locale +from urllib import quote +from email.utils import parsedate + + +def parse_datetime(string): + return datetime(*(parsedate(string)[:6])) + + +def parse_html_value(html): + + return html[html.find('>')+1:html.rfind('<')] + + +def parse_a_href(atag): + + start = atag.find('"') + 1 + end = atag.find('"', start) + return atag[start:end] + + +def convert_to_utf8_str(arg): + # written by Michael Norton (http://docondev.blogspot.com/) + if isinstance(arg, unicode): + arg = arg.encode('utf-8') + elif not isinstance(arg, str): + arg = str(arg) + return arg + + + +def import_simplejson(): + try: + import simplejson as json + except ImportError: + try: + import json # Python 2.6+ + except ImportError: + try: + from django.utils import simplejson as json # Google App Engine + except ImportError: + raise ImportError("Can't load a json library") + + return json + +def list_to_csv(item_list): + if item_list: + return ','.join([str(i) for i in item_list]) + +def urlencode_noplus(query): + return '&'.join(['%s=%s' % (quote(str(k), ''), quote(str(v), '')) \ + for k, v in query.iteritems()]) diff --git a/twitter-stream1.py b/twitter-stream1.py new file mode 100644 index 0000000..1127d4a --- /dev/null +++ b/twitter-stream1.py @@ -0,0 +1,20 @@ +import tweepy +from twitter_authentication import API_KEY, API_SECRET, ACCESS_TOKEN, ACCESS_TOKEN_SECRET + +auth = tweepy.OAuthHandler(API_KEY, API_SECRET) +auth.set_access_token(ACCESS_TOKEN, ACCESS_TOKEN_SECRET) + +api = tweepy.API(auth) + +class StreamListener(tweepy.StreamListener): + def on_status(self, tweet): + print tweet.user.screen_name + "\t" + tweet.text + + def on_error(self, status_code): + print 'Error: ' + repr(status_code) + return False + +l = StreamListener() +streamer = tweepy.Stream(auth=auth, listener=l) + +streamer.sample() diff --git a/twitter-stream2.py b/twitter-stream2.py new file mode 100644 index 0000000..59e0cf2 --- /dev/null +++ b/twitter-stream2.py @@ -0,0 +1,21 @@ +import tweepy +from twitter_authentication import API_KEY, API_SECRET, ACCESS_TOKEN, ACCESS_TOKEN_SECRET + +auth = tweepy.OAuthHandler(API_KEY, API_SECRET) +auth.set_access_token(ACCESS_TOKEN, ACCESS_TOKEN_SECRET) + +api = tweepy.API(auth) + +class StreamListener(tweepy.StreamListener): + def on_status(self, tweet): + print tweet.user.screen_name + "\t" + tweet.text + + def on_error(self, status_code): + print 'Error: ' + repr(status_code) + return False + +l = StreamListener() +streamer = tweepy.Stream(auth=auth, listener=l) + +keywords = ['python', 'perl'] +streamer.filter(track = keywords) diff --git a/twitter1.py b/twitter1.py new file mode 100644 index 0000000..fe0791d --- /dev/null +++ b/twitter1.py @@ -0,0 +1,12 @@ +import tweepy +from twitter_authentication import API_KEY, API_SECRET, ACCESS_TOKEN, ACCESS_TOKEN_SECRET + +auth = tweepy.OAuthHandler(API_KEY, API_SECRET) +auth.set_access_token(ACCESS_TOKEN, ACCESS_TOKEN_SECRET) + +api = tweepy.API(auth) + +public_tweets = api.home_timeline(count=100) + +for tweet in public_tweets: + print tweet.text diff --git a/twitter2.py b/twitter2.py new file mode 100644 index 0000000..44197e6 --- /dev/null +++ b/twitter2.py @@ -0,0 +1,17 @@ +import tweepy +from twitter_authentication import API_KEY, API_SECRET, ACCESS_TOKEN, ACCESS_TOKEN_SECRET + +auth = tweepy.OAuthHandler(API_KEY, API_SECRET) +auth.set_access_token(ACCESS_TOKEN, ACCESS_TOKEN_SECRET) + +api = tweepy.API(auth) + +user = api.get_user('makoshark') + +print user.screen_name + " has " + str(user.followers_count) + " followers." + +print "They include these 100 people:" + +for friend in user.friends(count=100): + print friend.screen_name + diff --git a/twitter3.py b/twitter3.py new file mode 100644 index 0000000..aced2be --- /dev/null +++ b/twitter3.py @@ -0,0 +1,12 @@ +import tweepy +from twitter_authentication import API_KEY, API_SECRET, ACCESS_TOKEN, ACCESS_TOKEN_SECRET + +auth = tweepy.OAuthHandler(API_KEY, API_SECRET) +auth.set_access_token(ACCESS_TOKEN, ACCESS_TOKEN_SECRET) + +api = tweepy.API(auth) + +public_tweets = api.search("data science", count=20) + +for tweet in public_tweets: + print tweet.user.screen_name + "\t" + str(tweet.created_at) + "\t" + tweet.text diff --git a/twitter4.py b/twitter4.py new file mode 100644 index 0000000..b588b1c --- /dev/null +++ b/twitter4.py @@ -0,0 +1,17 @@ +import tweepy +from twitter_authentication import API_KEY, API_SECRET, ACCESS_TOKEN, ACCESS_TOKEN_SECRET + +auth = tweepy.OAuthHandler(API_KEY, API_SECRET) +auth.set_access_token(ACCESS_TOKEN, ACCESS_TOKEN_SECRET) + +api = tweepy.API(auth) + +# code to write the file +import codecs +output_file = codecs.open("MY_DATA.tsv", "w", "utf-8") + +public_tweets = api.search("data science", count=10) + +for tweet in public_tweets: + print >>output_file, tweet.user.screen_name + "\t" + str(tweet.created_at) + "\t" + tweet.text + diff --git a/twitter_authentication.py b/twitter_authentication.py new file mode 100644 index 0000000..22655df --- /dev/null +++ b/twitter_authentication.py @@ -0,0 +1,4 @@ +API_KEY = 'CHANGE_ME' +API_SECRET = 'CHANGE_ME' +ACCESS_TOKEN = 'CHANGE_ME' +ACCESS_TOKEN_SECRET = 'CHANGE_ME' -- 2.30.2