From 43ab9572b404f122adb8de58bd03acd3f4f1a1c9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= <mike_faehrmann@web.de>
Date: Sat, 4 Jan 2020 23:46:29 +0100
Subject: [PATCH] [twitter] handle API rate limits (#526)

---
 gallery_dl/cache.py             |  2 +-
 gallery_dl/extractor/twitter.py | 37 +++++++++++++++++++++++++--------
 2 files changed, 29 insertions(+), 10 deletions(-)

diff --git a/gallery_dl/cache.py b/gallery_dl/cache.py
index 18241955..c48b53f0 100644
--- a/gallery_dl/cache.py
+++ b/gallery_dl/cache.py
@@ -37,7 +37,7 @@ class CacheDecorator():
     def update(self, key, value):
         self.cache[key] = value
 
-    def invalidate(self, key):
+    def invalidate(self, key=""):
         try:
             del self.cache[key]
         except KeyError:
diff --git a/gallery_dl/extractor/twitter.py b/gallery_dl/extractor/twitter.py
index a2dc047b..610e0eeb 100644
--- a/gallery_dl/extractor/twitter.py
+++ b/gallery_dl/extractor/twitter.py
@@ -57,6 +57,8 @@ class TwitterExtractor(Extractor):
                         self.root, data["tweet_id"])
                 else:
                     url = self._video_from_tweet(data["tweet_id"])
+                    if not url:
+                        continue
                     ext = text.ext_from_url(url)
                     if ext == "m3u8":
                         url = "ytdl:" + url
@@ -183,19 +185,28 @@ class TwitterExtractor(Extractor):
         if self.logged_in:
             headers["x-twitter-auth-type"] = "OAuth2Session"
         else:
-            token = self._guest_token(headers)
+            token = _guest_token(self, headers)
             cookies = {"gt": token}
             headers["x-guest-token"] = token
 
-        data = self.request(url, cookies=cookies, headers=headers).json()
-        return data["track"]["playbackUrl"]
+        response = self.request(
+            url, cookies=cookies, headers=headers, fatal=None)
 
-    @memcache()
-    def _guest_token(self, headers):
-        return self.request(
-            "https://api.twitter.com/1.1/guest/activate.json",
-            method="POST", headers=headers,
-        ).json().get("guest_token")
+        if response.status_code == 429 or \
+                response.headers.get("x-rate-limit-remaining") == "0":
+            if self.logged_in:
+                reset = response.headers.get("x-rate-limit-reset")
+                self.wait(until=reset, reason="rate limit reset")
+            else:
+                _guest_token.invalidate()
+            return self._video_from_tweet(tweet_id)
+
+        elif response.status_code >= 400:
+            self.log.warning("Unable to fetch video data for %s ('%s %s')",
+                             tweet_id, response.status_code, response.reason)
+            return None
+
+        return response.json()["track"]["playbackUrl"]
 
     def _tweets_from_api(self, url, max_position=None):
         params = {
@@ -357,3 +368,11 @@ class TwitterTweetExtractor(TwitterExtractor):
         end = page.index('class="js-tweet-stats-container')
         beg = page.rindex('<div class="tweet ', 0, end)
         return (page[beg:end],)
+
+
+@memcache()
+def _guest_token(extr, headers):
+    return extr.request(
+        "https://api.twitter.com/1.1/guest/activate.json",
+        method="POST", headers=headers,
+    ).json().get("guest_token")