Add basic rate limiter for Mastodon API calls

It seems at leaast mastodon.social may have implemented rate limiting at
the TCP level (afaict). To work aroud this, implement a 15 second rate
limit for Mastodon API calls to individual providers (allow other calls
to go quicker). As this is done per-process it only affects batch jobs,
but those are also the only ones with the issue.

Uncertain if this will actually work since the limit isn't documented
(unlike the http level limit), but it's worth a try.

2 files changed, 24 insertions, 1 deletions

diff --git a/postgresqleu/util/messaging/mastodon.py b/postgresqleu/util/messaging/mastodon.py
index e21b5f86..6d524b9e 100644
--- a/postgresqleu/util/messaging/mastodon.py
+++ b/postgresqleu/util/messaging/mastodon.py
@@ -17,7 +17,7 @@ from postgresqleu.util.messaging import re_token
 from postgresqleu.confreg.backendforms import BackendSeriesMessagingForm
 from postgresqleu.confreg.models import ConferenceRegistration, IncomingDirectMessage
 
-from .util import send_reg_direct_message
+from .util import send_reg_direct_message, ratelimiter
 from .common import register_messaging_config
 
 
@@ -155,9 +155,11 @@ class Mastodon(object):
         return '{}{}'.format(self.providerconfig['baseurl'], url)
 
     def _get(self, url, *args, **kwargs):
+        ratelimiter.limit(self.providerconfig['baseurl'])
         return self.sess.get(self._api_url(url), timeout=30, *args, **kwargs)
 
     def _post(self, url, *args, **kwargs):
+        ratelimiter.limit(self.providerconfig['baseurl'])
         return self.sess.post(self._api_url(url), timeout=30, *args, **kwargs)
 
     def get_account_info(self):
diff --git a/postgresqleu/util/messaging/util.py b/postgresqleu/util/messaging/util.py
index 3f0d899c..b78e8ae1 100644
--- a/postgresqleu/util/messaging/util.py
+++ b/postgresqleu/util/messaging/util.py
@@ -1,6 +1,7 @@
 from django.utils import timezone
 
 from datetime import timedelta
+import time
 
 from postgresqleu.confreg.models import NotificationQueue
 from postgresqleu.util.db import exec_no_result
@@ -82,3 +83,23 @@ def send_channel_message(messaging, channel, msg, expiry=timedelta(hours=1)):
 def notify_twitter_moderation(tweet, completed, approved):
     for messaging in tweet.conference.conferencemessaging_set.filter(socialmediamanagement=True, provider__active=True):
         get_messaging_class(messaging.provider.classname)(messaging.provider.id, messaging.provider.config).notify_twitter_moderation(messaging, tweet, completed, approved)
+
+
+# Some Mastodon servers have started doing rate limiting at the TCP level, it seems. So
+# we create a generic rate limiting class. Note that it will only rate limit within the
+# same process, but this problem only shows up in batch jobs so it should be fine.
+class _RateLimiter:
+    def __init__(self):
+        self.lastcalls = {}
+
+    def limit(self, baseurl):
+        if baseurl in self.lastcalls:
+            # Space calls out by 15 seconds per baseurl
+            s = 15 + self.lastcalls[baseurl] - time.time()
+            if s > 0:
+                time.sleep(s)
+
+        self.lastcalls[baseurl] = time.time()
+
+
+ratelimiter = _RateLimiter()