throttle updates if received HTTP 429 (Too Many Requests)
This commit is contained in:
parent
7cd26272fa
commit
2975c7297b
|
@ -189,6 +189,9 @@ class Config {
|
|||
/** http user agent (changing this is not recommended) */
|
||||
const HTTP_USER_AGENT = "HTTP_USER_AGENT";
|
||||
|
||||
/** delay updates for this feed if received HTTP 429 (Too Many Requests) for this amount of seconds (base value, actual delay is base...base*2) */
|
||||
const HTTP_429_THROTTLE_INTERVAL = "HTTP_429_THROTTLE_INTERVAL";
|
||||
|
||||
/** default values for all global configuration options */
|
||||
private const _DEFAULTS = [
|
||||
Config::DB_TYPE => [ "pgsql", Config::T_STRING ],
|
||||
|
@ -245,6 +248,7 @@ class Config {
|
|||
Config::AUTH_MIN_INTERVAL => [ 5, Config::T_INT ],
|
||||
Config::HTTP_USER_AGENT => [ 'Tiny Tiny RSS/%s (https://tt-rss.org/)',
|
||||
Config::T_STRING ],
|
||||
Config::HTTP_429_THROTTLE_INTERVAL => [ 3600, Config::T_INT ],
|
||||
];
|
||||
|
||||
/** @var Config|null */
|
||||
|
|
|
@ -17,8 +17,12 @@ class Db
|
|||
}
|
||||
}
|
||||
|
||||
static function NOW(): string {
|
||||
return date("Y-m-d H:i:s", time());
|
||||
/**
|
||||
* @param int $delta adjust generated timestamp by this value in seconds (either positive or negative)
|
||||
* @return string
|
||||
*/
|
||||
static function NOW(int $delta = 0): string {
|
||||
return date("Y-m-d H:i:s", time() + $delta);
|
||||
}
|
||||
|
||||
private function __clone() {
|
||||
|
|
|
@ -492,7 +492,7 @@ class RSSUtils {
|
|||
|
||||
// If-Modified-Since
|
||||
if (UrlHelper::$fetch_last_error_code == 304) {
|
||||
Debug::log("source claims data not modified, nothing to do.", Debug::LOG_VERBOSE);
|
||||
Debug::log("source claims data not modified (304), nothing to do.", Debug::LOG_VERBOSE);
|
||||
$error_message = "";
|
||||
|
||||
$feed_obj->set([
|
||||
|
@ -503,6 +503,24 @@ class RSSUtils {
|
|||
|
||||
$feed_obj->save();
|
||||
|
||||
} else if (UrlHelper::$fetch_last_error_code == 429) {
|
||||
|
||||
// randomize interval using Config::HTTP_429_THROTTLE_INTERVAL as a base value (1-2x)
|
||||
$http_429_throttle_interval = rand(Config::get(Config::HTTP_429_THROTTLE_INTERVAL),
|
||||
Config::get(Config::HTTP_429_THROTTLE_INTERVAL)*2);
|
||||
|
||||
$error_message = UrlHelper::$fetch_last_error;
|
||||
|
||||
Debug::log("source claims we're requesting too often (429), throttling updates for $http_429_throttle_interval seconds.",
|
||||
Debug::LOG_VERBOSE);
|
||||
|
||||
$feed_obj->set([
|
||||
'last_error' => $error_message . " (updates throttled for $http_429_throttle_interval seconds.)",
|
||||
'last_successful_update' => Db::NOW($http_429_throttle_interval),
|
||||
'last_updated' => Db::NOW($http_429_throttle_interval),
|
||||
]);
|
||||
|
||||
$feed_obj->save();
|
||||
} else {
|
||||
$error_message = UrlHelper::$fetch_last_error;
|
||||
|
||||
|
|
Loading…
Reference in New Issue