valvin
/
ttrss
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

throttle updates if received HTTP 429 (Too Many Requests)

This commit is contained in:
Andrew Dolgov 2022-06-09 09:05:36 +03:00
parent 7cd26272fa
commit 2975c7297b
No known key found for this signature in database
GPG Key ID: 1A56B4FA25D4AF2A
3 changed files with 29 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
classes/config.php
View File

@ -189,6 +189,9 @@ class Config {
/** http user agent (changing this is not recommended) */ /** http user agent (changing this is not recommended) */
const HTTP_USER_AGENT = "HTTP_USER_AGENT"; const HTTP_USER_AGENT = "HTTP_USER_AGENT";
/** delay updates for this feed if received HTTP 429 (Too Many Requests) for this amount of seconds (base value, actual delay is base...base*2) */
const HTTP_429_THROTTLE_INTERVAL = "HTTP_429_THROTTLE_INTERVAL";
/** default values for all global configuration options */ /** default values for all global configuration options */
private const _DEFAULTS = [ private const _DEFAULTS = [
Config::DB_TYPE => [ "pgsql", Config::T_STRING ], Config::DB_TYPE => [ "pgsql", Config::T_STRING ],
@ -245,6 +248,7 @@ class Config {
Config::AUTH_MIN_INTERVAL => [ 5, Config::T_INT ], Config::AUTH_MIN_INTERVAL => [ 5, Config::T_INT ],
Config::HTTP_USER_AGENT => [ 'Tiny Tiny RSS/%s (https://tt-rss.org/)', Config::HTTP_USER_AGENT => [ 'Tiny Tiny RSS/%s (https://tt-rss.org/)',
Config::T_STRING ], Config::T_STRING ],
Config::HTTP_429_THROTTLE_INTERVAL => [ 3600, Config::T_INT ],
]; ];
/** @var Config|null */ /** @var Config|null */

8
classes/db.php
View File

@ -17,8 +17,12 @@ class Db
} }
} }
static function NOW(): string { /**
return date("Y-m-d H:i:s", time()); * @param int $delta adjust generated timestamp by this value in seconds (either positive or negative)
* @return string
*/
static function NOW(int $delta = 0): string {
return date("Y-m-d H:i:s", time() + $delta);
} }
private function __clone() { private function __clone() {

20
classes/rssutils.php
View File

@ -492,7 +492,7 @@ class RSSUtils {
// If-Modified-Since // If-Modified-Since
if (UrlHelper::$fetch_last_error_code == 304) { if (UrlHelper::$fetch_last_error_code == 304) {
Debug::log("source claims data not modified, nothing to do.", Debug::LOG_VERBOSE); Debug::log("source claims data not modified (304), nothing to do.", Debug::LOG_VERBOSE);
$error_message = ""; $error_message = "";
$feed_obj->set([ $feed_obj->set([
@ -503,6 +503,24 @@ class RSSUtils {
$feed_obj->save(); $feed_obj->save();
} else if (UrlHelper::$fetch_last_error_code == 429) {
// randomize interval using Config::HTTP_429_THROTTLE_INTERVAL as a base value (1-2x)
$http_429_throttle_interval = rand(Config::get(Config::HTTP_429_THROTTLE_INTERVAL),
Config::get(Config::HTTP_429_THROTTLE_INTERVAL)*2);
$error_message = UrlHelper::$fetch_last_error;
Debug::log("source claims we're requesting too often (429), throttling updates for $http_429_throttle_interval seconds.",
Debug::LOG_VERBOSE);
$feed_obj->set([
'last_error' => $error_message . " (updates throttled for $http_429_throttle_interval seconds.)",
'last_successful_update' => Db::NOW($http_429_throttle_interval),
'last_updated' => Db::NOW($http_429_throttle_interval),
]);
$feed_obj->save();
} else { } else {
$error_message = UrlHelper::$fetch_last_error; $error_message = UrlHelper::$fetch_last_error;