Fixed key error in some tweets

[twitter-api-cdsw] / tweepy / cursor.py

diff --git a/tweepy/cursor.py b/tweepy/cursor.py
index 4c06f17a9f2fab3af9885f831c17c2c7fde1152a..1d63f4979deb67137bc6e1dc3e2b8ef667e468fa 100644 (file)
--- a/tweepy/cursor.py
+++ b/tweepy/cursor.py
@@ -2,7 +2,11 @@
 # Copyright 2009-2010 Joshua Roesslein
 # See LICENSE for details.
 
 # Copyright 2009-2010 Joshua Roesslein
 # See LICENSE for details.
 

+from __future__ import print_function
+

 from tweepy.error import TweepError
 from tweepy.error import TweepError

+from tweepy.parsers import ModelParser, RawParser
+

 
 class Cursor(object):
     """Pagination helper class"""
 
 class Cursor(object):
     """Pagination helper class"""


@@ -32,6 +36,7 @@ class Cursor(object):
         i.limit = limit
         return i
 
         i.limit = limit
         return i
 

+

 class BaseIterator(object):
 
     def __init__(self, method, args, kargs):
 class BaseIterator(object):
 
     def __init__(self, method, args, kargs):


@@ -40,6 +45,9 @@ class BaseIterator(object):
         self.kargs = kargs
         self.limit = 0
 
         self.kargs = kargs
         self.limit = 0
 

+    def __next__(self):
+        return self.next()
+

     def next(self):
         raise NotImplementedError
 
     def next(self):
         raise NotImplementedError
 


@@ -49,6 +57,7 @@ class BaseIterator(object):
     def __iter__(self):
         return self
 
     def __iter__(self):
         return self
 

+

 class CursorIterator(BaseIterator):
 
     def __init__(self, method, args, kargs):
 class CursorIterator(BaseIterator):
 
     def __init__(self, method, args, kargs):


@@ -56,67 +65,96 @@ class CursorIterator(BaseIterator):
         start_cursor = kargs.pop('cursor', None)
         self.next_cursor = start_cursor or -1
         self.prev_cursor = start_cursor or 0
         start_cursor = kargs.pop('cursor', None)
         self.next_cursor = start_cursor or -1
         self.prev_cursor = start_cursor or 0

-        self.count = 0
+        self.num_tweets = 0

 
     def next(self):
 
     def next(self):

-        if self.next_cursor == 0 or (self.limit and self.count == self.limit):
+        if self.next_cursor == 0 or (self.limit and self.num_tweets == self.limit):

             raise StopIteration
             raise StopIteration

-        data, cursors = self.method(
-                cursor=self.next_cursor, *self.args, **self.kargs
-        )
+        data, cursors = self.method(cursor=self.next_cursor,
+                                    *self.args,
+                                    **self.kargs)

         self.prev_cursor, self.next_cursor = cursors
         if len(data) == 0:
             raise StopIteration
         self.prev_cursor, self.next_cursor = cursors
         if len(data) == 0:
             raise StopIteration

-        self.count += 1
+        self.num_tweets += 1

         return data
 
     def prev(self):
         if self.prev_cursor == 0:
             raise TweepError('Can not page back more, at first page')
         return data
 
     def prev(self):
         if self.prev_cursor == 0:
             raise TweepError('Can not page back more, at first page')

-        data, self.next_cursor, self.prev_cursor = self.method(
-                cursor=self.prev_cursor, *self.args, **self.kargs
-        )
-        self.count -= 1
+        data, self.next_cursor, self.prev_cursor = self.method(cursor=self.prev_cursor,
+                                                               *self.args,
+                                                               **self.kargs)
+        self.num_tweets -= 1

         return data
 
         return data
 

+

 class IdIterator(BaseIterator):
 
     def __init__(self, method, args, kargs):
         BaseIterator.__init__(self, method, args, kargs)
 class IdIterator(BaseIterator):
 
     def __init__(self, method, args, kargs):
         BaseIterator.__init__(self, method, args, kargs)

-        self.max_id = kargs.get('max_id')
-        self.since_id = kargs.get('since_id')
-        self.count = 0
+        self.max_id = kargs.pop('max_id', None)
+        self.num_tweets = 0
+        self.results = []
+        self.model_results = []
+        self.index = 0

 
     def next(self):
         """Fetch a set of items with IDs less than current set."""
 
     def next(self):
         """Fetch a set of items with IDs less than current set."""

-        if self.limit and self.limit == self.count:
+        if self.limit and self.limit == self.num_tweets:

             raise StopIteration
 
             raise StopIteration
 

-        # max_id is inclusive so decrement by one
-        # to avoid requesting duplicate items.
-        max_id = self.since_id - 1 if self.max_id else None
-        data = self.method(max_id = max_id, *self.args, **self.kargs)
-        if len(data) == 0:
+        if self.index >= len(self.results) - 1:
+            data = self.method(max_id=self.max_id, parser=RawParser(), *self.args, **self.kargs)
+
+            if hasattr(self.method, '__self__'):
+                old_parser = self.method.__self__.parser
+                # Hack for models which expect ModelParser to be set
+                self.method.__self__.parser = ModelParser()
+
+            # This is a special invocation that returns the underlying
+            # APIMethod class
+            model = ModelParser().parse(self.method(create=True), data)
+            if hasattr(self.method, '__self__'):
+                self.method.__self__.parser = old_parser
+                result = self.method.__self__.parser.parse(self.method(create=True), data)
+            else:
+                result = model
+
+            if len(self.results) != 0:
+                self.index += 1
+            self.results.append(result)
+            self.model_results.append(model)
+        else:
+            self.index += 1
+            result = self.results[self.index]
+            model = self.model_results[self.index]
+
+        if len(result) == 0:

             raise StopIteration
             raise StopIteration

-        self.max_id = data.max_id
-        self.since_id = data.since_id
-        self.count += 1
-        return data
+        # TODO: Make this not dependant on the parser making max_id and
+        # since_id available
+        self.max_id = model.max_id
+        self.num_tweets += 1
+        return result

 
     def prev(self):
         """Fetch a set of items with IDs greater than current set."""
 
     def prev(self):
         """Fetch a set of items with IDs greater than current set."""

-        if self.limit and self.limit == self.count:
+        if self.limit and self.limit == self.num_tweets:

             raise StopIteration
 
             raise StopIteration
 

-        since_id = self.max_id
-        data = self.method(since_id = since_id, *self.args, **self.kargs)
-        if len(data) == 0:
+        self.index -= 1
+        if self.index < 0:
+            # There's no way to fetch a set of tweets directly 'above' the
+            # current set

             raise StopIteration
             raise StopIteration

-        self.max_id = data.max_id
-        self.since_id = data.since_id
-        self.count += 1
+
+        data = self.results[self.index]
+        self.max_id = self.model_results[self.index].max_id
+        self.num_tweets += 1

         return data
 
         return data
 

+

 class PageIterator(BaseIterator):
 
     def __init__(self, method, args, kargs):
 class PageIterator(BaseIterator):
 
     def __init__(self, method, args, kargs):


@@ -124,18 +162,23 @@ class PageIterator(BaseIterator):
         self.current_page = 0
 
     def next(self):
         self.current_page = 0
 
     def next(self):

-        self.current_page += 1
+        if self.limit > 0:
+            if self.current_page > self.limit:
+                raise StopIteration
+

         items = self.method(page=self.current_page, *self.args, **self.kargs)
         items = self.method(page=self.current_page, *self.args, **self.kargs)

-        if len(items) == 0 or (self.limit > 0 and self.current_page > self.limit):
+        if len(items) == 0:

             raise StopIteration
             raise StopIteration

+        self.current_page += 1

         return items
 
     def prev(self):
         return items
 
     def prev(self):

-        if (self.current_page == 1):
+        if self.current_page == 1:

             raise TweepError('Can not page back more, at first page')
         self.current_page -= 1
         return self.method(page=self.current_page, *self.args, **self.kargs)
 
             raise TweepError('Can not page back more, at first page')
         self.current_page -= 1
         return self.method(page=self.current_page, *self.args, **self.kargs)
 

+

 class ItemIterator(BaseIterator):
 
     def __init__(self, page_iterator):
 class ItemIterator(BaseIterator):
 
     def __init__(self, page_iterator):


@@ -143,17 +186,18 @@ class ItemIterator(BaseIterator):
         self.limit = 0
         self.current_page = None
         self.page_index = -1
         self.limit = 0
         self.current_page = None
         self.page_index = -1

-        self.count = 0
+        self.num_tweets = 0

 
     def next(self):
 
     def next(self):

-        if self.limit > 0 and self.count == self.limit:
-            raise StopIteration
+        if self.limit > 0:
+            if self.num_tweets == self.limit:
+                raise StopIteration

         if self.current_page is None or self.page_index == len(self.current_page) - 1:
             # Reached end of current page, get the next page...
             self.current_page = self.page_iterator.next()
             self.page_index = -1
         self.page_index += 1
         if self.current_page is None or self.page_index == len(self.current_page) - 1:
             # Reached end of current page, get the next page...
             self.current_page = self.page_iterator.next()
             self.page_index = -1
         self.page_index += 1

-        self.count += 1
+        self.num_tweets += 1

         return self.current_page[self.page_index]
 
     def prev(self):
         return self.current_page[self.page_index]
 
     def prev(self):


@@ -166,6 +210,5 @@ class ItemIterator(BaseIterator):
             if self.page_index == 0:
                 raise TweepError('No more items')
         self.page_index -= 1
             if self.page_index == 0:
                 raise TweepError('No more items')
         self.page_index -= 1

-        self.count -= 1
+        self.num_tweets -= 1

         return self.current_page[self.page_index]
         return self.current_page[self.page_index]

-