From 2975c7297b680e486f326939b9fba82d8cf18035 Mon Sep 17 00:00:00 2001
From: Andrew Dolgov <noreply@fakecake.org>
Date: Thu, 9 Jun 2022 09:05:36 +0300
Subject: [PATCH] throttle updates if received HTTP 429 (Too Many Requests)

---
 classes/config.php   |  4 ++++
 classes/db.php       |  8 ++++++--
 classes/rssutils.php | 20 +++++++++++++++++++-
 3 files changed, 29 insertions(+), 3 deletions(-)

diff --git a/classes/config.php b/classes/config.php
index cc089b7ba..74546e3f2 100644
--- a/classes/config.php
+++ b/classes/config.php
@@ -189,6 +189,9 @@ class Config {
 	/** http user agent (changing this is not recommended) */
 	const HTTP_USER_AGENT = "HTTP_USER_AGENT";
 
+	/** delay updates for this feed if received HTTP 429 (Too Many Requests) for this amount of seconds (base value, actual delay is base...base*2) */
+	const HTTP_429_THROTTLE_INTERVAL = "HTTP_429_THROTTLE_INTERVAL";
+
 	/** default values for all global configuration options */
 	private const _DEFAULTS = [
 		Config::DB_TYPE => [ "pgsql", 									Config::T_STRING ],
@@ -245,6 +248,7 @@ class Config {
 		Config::AUTH_MIN_INTERVAL => [ 5,								Config::T_INT ],
 		Config::HTTP_USER_AGENT => [ 'Tiny Tiny RSS/%s (https://tt-rss.org/)',
 																					Config::T_STRING ],
+		Config::HTTP_429_THROTTLE_INTERVAL => [ 3600,				Config::T_INT ],
 	];
 
 	/** @var Config|null */
diff --git a/classes/db.php b/classes/db.php
index 2cc89f5ba..4331b662e 100755
--- a/classes/db.php
+++ b/classes/db.php
@@ -17,8 +17,12 @@ class Db
 		}
 	}
 
-	static function NOW(): string {
-		return date("Y-m-d H:i:s", time());
+	/**
+	 * @param int $delta adjust generated timestamp by this value in seconds (either positive or negative)
+	 * @return string
+	 */
+	static function NOW(int $delta = 0): string {
+		return date("Y-m-d H:i:s", time() + $delta);
 	}
 
 	private function __clone() {
diff --git a/classes/rssutils.php b/classes/rssutils.php
index aec17d538..1d87e73d6 100755
--- a/classes/rssutils.php
+++ b/classes/rssutils.php
@@ -492,7 +492,7 @@ class RSSUtils {
 
 			// If-Modified-Since
 			if (UrlHelper::$fetch_last_error_code == 304) {
-				Debug::log("source claims data not modified, nothing to do.", Debug::LOG_VERBOSE);
+				Debug::log("source claims data not modified (304), nothing to do.", Debug::LOG_VERBOSE);
 				$error_message = "";
 
 				$feed_obj->set([
@@ -503,6 +503,24 @@ class RSSUtils {
 
 				$feed_obj->save();
 
+			} else if (UrlHelper::$fetch_last_error_code == 429) {
+
+				// randomize interval using Config::HTTP_429_THROTTLE_INTERVAL as a base value (1-2x)
+				$http_429_throttle_interval = rand(Config::get(Config::HTTP_429_THROTTLE_INTERVAL),
+					Config::get(Config::HTTP_429_THROTTLE_INTERVAL)*2);
+
+				$error_message = UrlHelper::$fetch_last_error;
+
+				Debug::log("source claims we're requesting too often (429), throttling updates for $http_429_throttle_interval seconds.",
+					Debug::LOG_VERBOSE);
+
+				$feed_obj->set([
+					'last_error' => $error_message . " (updates throttled for $http_429_throttle_interval seconds.)",
+					'last_successful_update' => Db::NOW($http_429_throttle_interval),
+					'last_updated' => Db::NOW($http_429_throttle_interval),
+				]);
+
+				$feed_obj->save();
 			} else {
 				$error_message = UrlHelper::$fetch_last_error;