deal with feed update scheduling w/ new prefs

This commit is contained in:
Andrew Dolgov 2021-02-25 14:20:54 +03:00
parent 00d0cb8c81
commit 7869378436
3 changed files with 91 additions and 65 deletions

View File

@ -147,6 +147,13 @@ class Prefs {
return isset(self::_DEFAULTS[$pref_name]); return isset(self::_DEFAULTS[$pref_name]);
} }
static function get_default(string $pref_name) {
if (self::is_valid($pref_name))
return self::_DEFAULTS[$pref_name][0];
else
return null;
}
function __construct() { function __construct() {
$this->pdo = Db::pdo(); $this->pdo = Db::pdo();

View File

@ -200,36 +200,42 @@ class RPC extends Handler_Protected {
static function updaterandomfeed_real() { static function updaterandomfeed_real() {
$default_interval = (int) Prefs::get_default(Prefs::DEFAULT_UPDATE_INTERVAL);
// Test if the feed need a update (update interval exceded). // Test if the feed need a update (update interval exceded).
if (Config::get(Config::DB_TYPE) == "pgsql") { if (Config::get(Config::DB_TYPE) == "pgsql") {
$update_limit_qpart = "AND (( $update_limit_qpart = "AND ((
ttrss_feeds.update_interval = 0 update_interval = 0
AND ttrss_feeds.last_updated < NOW() - CAST((ttrss_user_prefs.value || ' minutes') AS INTERVAL) AND p.value != '-1'
AND last_updated < NOW() - CAST((COALESCE(p.value, '$default_interval') || ' minutes') AS INTERVAL)
) OR ( ) OR (
ttrss_feeds.update_interval > 0 update_interval > 0
AND ttrss_feeds.last_updated < NOW() - CAST((ttrss_feeds.update_interval || ' minutes') AS INTERVAL) AND last_updated < NOW() - CAST((update_interval || ' minutes') AS INTERVAL)
) OR ( ) OR (
ttrss_feeds.update_interval >= 0 update_interval >= 0
AND p.value != '-1'
AND (last_updated = '1970-01-01 00:00:00' OR last_updated IS NULL) AND (last_updated = '1970-01-01 00:00:00' OR last_updated IS NULL)
))"; ))";
} else { } else {
$update_limit_qpart = "AND (( $update_limit_qpart = "AND ((
ttrss_feeds.update_interval = 0 update_interval = 0
AND ttrss_feeds.last_updated < DATE_SUB(NOW(), INTERVAL CONVERT(ttrss_user_prefs.value, SIGNED INTEGER) MINUTE) AND p.value != '-1'
AND last_updated < DATE_SUB(NOW(), INTERVAL CONVERT(COALESCE(p.value, '$default_interval'), SIGNED INTEGER) MINUTE)
) OR ( ) OR (
ttrss_feeds.update_interval > 0 update_interval > 0
AND ttrss_feeds.last_updated < DATE_SUB(NOW(), INTERVAL ttrss_feeds.update_interval MINUTE) AND last_updated < DATE_SUB(NOW(), INTERVAL update_interval MINUTE)
) OR ( ) OR (
ttrss_feeds.update_interval >= 0 update_interval >= 0
AND p.value != '-1'
AND (last_updated = '1970-01-01 00:00:00' OR last_updated IS NULL) AND (last_updated = '1970-01-01 00:00:00' OR last_updated IS NULL)
))"; ))";
} }
// Test if feed is currently being updated by another process. // Test if feed is currently being updated by another process.
if (Config::get(Config::DB_TYPE) == "pgsql") { if (Config::get(Config::DB_TYPE) == "pgsql") {
$updstart_thresh_qpart = "AND (ttrss_feeds.last_update_started IS NULL OR ttrss_feeds.last_update_started < NOW() - INTERVAL '5 minutes')"; $updstart_thresh_qpart = "AND (last_update_started IS NULL OR last_update_started < NOW() - INTERVAL '5 minutes')";
} else { } else {
$updstart_thresh_qpart = "AND (ttrss_feeds.last_update_started IS NULL OR ttrss_feeds.last_update_started < DATE_SUB(NOW(), INTERVAL 5 MINUTE))"; $updstart_thresh_qpart = "AND (last_update_started IS NULL OR last_update_started < DATE_SUB(NOW(), INTERVAL 5 MINUTE))";
} }
$random_qpart = Db::sql_random_function(); $random_qpart = Db::sql_random_function();
@ -237,24 +243,24 @@ class RPC extends Handler_Protected {
$pdo = Db::pdo(); $pdo = Db::pdo();
// we could be invoked from public.php with no active session // we could be invoked from public.php with no active session
if ($_SESSION["uid"]) { if (!empty($_SESSION["uid"])) {
$owner_check_qpart = "AND ttrss_feeds.owner_uid = ".$pdo->quote($_SESSION["uid"]); $owner_check_qpart = "AND f.owner_uid = ".$pdo->quote($_SESSION["uid"]);
} else { } else {
$owner_check_qpart = ""; $owner_check_qpart = "";
} }
// We search for feed needing update. $query = "SELECT f.feed_url,f.id
$res = $pdo->query("SELECT ttrss_feeds.feed_url,ttrss_feeds.id
FROM FROM
ttrss_feeds, ttrss_users, ttrss_user_prefs ttrss_feeds f, ttrss_users u LEFT JOIN ttrss_user_prefs2 p ON
(p.owner_uid = u.id AND profile IS NULL AND pref_name = 'DEFAULT_UPDATE_INTERVAL')
WHERE WHERE
ttrss_feeds.owner_uid = ttrss_users.id f.owner_uid = u.id
AND ttrss_users.id = ttrss_user_prefs.owner_uid
AND ttrss_user_prefs.pref_name = 'DEFAULT_UPDATE_INTERVAL'
$owner_check_qpart $owner_check_qpart
$update_limit_qpart $update_limit_qpart
$updstart_thresh_qpart $updstart_thresh_qpart
ORDER BY $random_qpart LIMIT 30"); ORDER BY $random_qpart LIMIT 30";
$res = $pdo->query($query);
$num_updated = 0; $num_updated = 0;

View File

@ -64,37 +64,45 @@ class RSSUtils {
$pdo = Db::pdo(); $pdo = Db::pdo();
if (!Config::get(Config::SINGLE_USER_MODE) && Config::get(Config::DAEMON_UPDATE_LOGIN_LIMIT) > 0) { if (!Config::get(Config::SINGLE_USER_MODE) && Config::get(Config::DAEMON_UPDATE_LOGIN_LIMIT) > 0) {
$login_limit = (int) Config::get(Config::DAEMON_UPDATE_LOGIN_LIMIT);
if (Config::get(Config::DB_TYPE) == "pgsql") { if (Config::get(Config::DB_TYPE) == "pgsql") {
$login_thresh_qpart = "AND ttrss_users.last_login >= NOW() - INTERVAL '".Config::get(Config::DAEMON_UPDATE_LOGIN_LIMIT)." days'"; $login_thresh_qpart = "AND last_login >= NOW() - INTERVAL '$login_limit days'";
} else { } else {
$login_thresh_qpart = "AND ttrss_users.last_login >= DATE_SUB(NOW(), INTERVAL ".Config::get(Config::DAEMON_UPDATE_LOGIN_LIMIT)." DAY)"; $login_thresh_qpart = "AND last_login >= DATE_SUB(NOW(), INTERVAL $login_limit DAY)";
} }
} else { } else {
$login_thresh_qpart = ""; $login_thresh_qpart = "";
} }
$default_interval = (int) Prefs::get_default(Prefs::DEFAULT_UPDATE_INTERVAL);
if (Config::get(Config::DB_TYPE) == "pgsql") { if (Config::get(Config::DB_TYPE) == "pgsql") {
$update_limit_qpart = "AND (( $update_limit_qpart = "AND ((
ttrss_feeds.update_interval = 0 update_interval = 0
AND ttrss_user_prefs.value != '-1' AND p.value != '-1'
AND last_updated < NOW() - CAST((ttrss_user_prefs.value || ' minutes') AS INTERVAL) AND last_updated < NOW() - CAST((COALESCE(p.value, '$default_interval') || ' minutes') AS INTERVAL)
) OR ( ) OR (
ttrss_feeds.update_interval > 0 update_interval > 0
AND last_updated < NOW() - CAST((ttrss_feeds.update_interval || ' minutes') AS INTERVAL) AND last_updated < NOW() - CAST((update_interval || ' minutes') AS INTERVAL)
) OR ((last_updated = '1970-01-01 00:00:00' OR last_updated IS NULL) ) OR (
AND ttrss_feeds.update_interval >= 0 update_interval >= 0
AND ttrss_user_prefs.value != '-1'))"; AND p.value != '-1'
AND (last_updated = '1970-01-01 00:00:00' OR last_updated IS NULL)
))";
} else { } else {
$update_limit_qpart = "AND (( $update_limit_qpart = "AND ((
ttrss_feeds.update_interval = 0 update_interval = 0
AND ttrss_user_prefs.value != '-1' AND p.value != '-1'
AND last_updated < DATE_SUB(NOW(), INTERVAL CONVERT(ttrss_user_prefs.value, SIGNED INTEGER) MINUTE) AND last_updated < DATE_SUB(NOW(), INTERVAL CONVERT(COALESCE(p.value, '$default_interval'), SIGNED INTEGER) MINUTE)
) OR ( ) OR (
ttrss_feeds.update_interval > 0 update_interval > 0
AND last_updated < DATE_SUB(NOW(), INTERVAL ttrss_feeds.update_interval MINUTE) AND last_updated < DATE_SUB(NOW(), INTERVAL update_interval MINUTE)
) OR ((last_updated = '1970-01-01 00:00:00' OR last_updated IS NULL) ) OR (
AND ttrss_feeds.update_interval >= 0 update_interval >= 0
AND ttrss_user_prefs.value != '-1'))"; AND p.value != '-1'
AND (last_updated = '1970-01-01 00:00:00' OR last_updated IS NULL)
))";
} }
// Test if feed is currently being updated by another process. // Test if feed is currently being updated by another process.
@ -108,20 +116,23 @@ class RSSUtils {
// Update the least recently updated feeds first // Update the least recently updated feeds first
$query_order = "ORDER BY last_updated"; $query_order = "ORDER BY last_updated";
if (Config::get(Config::DB_TYPE) == "pgsql") $query_order .= " NULLS FIRST";
$query = "SELECT DISTINCT ttrss_feeds.feed_url, ttrss_feeds.last_updated if (Config::get(Config::DB_TYPE) == "pgsql")
$query_order .= " NULLS FIRST";
$query = "SELECT f.feed_url, f.last_updated
FROM FROM
ttrss_feeds, ttrss_users, ttrss_user_prefs ttrss_feeds f, ttrss_users u LEFT JOIN ttrss_user_prefs2 p ON
(p.owner_uid = u.id AND profile IS NULL AND pref_name = 'DEFAULT_UPDATE_INTERVAL')
WHERE WHERE
ttrss_feeds.owner_uid = ttrss_users.id f.owner_uid = u.id
AND ttrss_user_prefs.profile IS NULL $login_thresh_qpart
AND ttrss_users.id = ttrss_user_prefs.owner_uid $update_limit_qpart
AND ttrss_user_prefs.pref_name = 'DEFAULT_UPDATE_INTERVAL'
$login_thresh_qpart $update_limit_qpart
$updstart_thresh_qpart $updstart_thresh_qpart
$query_order $query_limit"; $query_order $query_limit";
//print "$query\n";
$res = $pdo->query($query); $res = $pdo->query($query);
$feeds_to_update = array(); $feeds_to_update = array();
@ -144,34 +155,36 @@ class RSSUtils {
$nf = 0; $nf = 0;
$bstarted = microtime(true); $bstarted = microtime(true);
$batch_owners = array(); $batch_owners = [];
// since we have the data cached, we can deal with other feeds with the same url $user_query = "SELECT f.id,
$usth = $pdo->prepare("SELECT
DISTINCT ttrss_feeds.id,
last_updated, last_updated,
ttrss_feeds.owner_uid, f.owner_uid,
ttrss_feeds.title f.title
FROM ttrss_feeds, ttrss_users, ttrss_user_prefs WHERE FROM ttrss_feeds f, ttrss_users u LEFT JOIN ttrss_user_prefs2 p ON
ttrss_user_prefs.owner_uid = ttrss_feeds.owner_uid AND (p.owner_uid = u.id AND profile IS NULL AND pref_name = 'DEFAULT_UPDATE_INTERVAL')
ttrss_users.id = ttrss_user_prefs.owner_uid AND WHERE
ttrss_user_prefs.pref_name = 'DEFAULT_UPDATE_INTERVAL' AND f.owner_uid = u.id
ttrss_user_prefs.profile IS NULL AND AND feed_url = :feed
feed_url = ?
$update_limit_qpart
$login_thresh_qpart $login_thresh_qpart
ORDER BY ttrss_feeds.id $query_limit"); $update_limit_qpart
ORDER BY f.id $query_limit";
//print "$user_query\n";
// since we have feed xml cached, we can deal with other feeds with the same url
$usth = $pdo->prepare($user_query);
foreach ($feeds_to_update as $feed) { foreach ($feeds_to_update as $feed) {
Debug::log("Base feed: $feed"); Debug::log("Base feed: $feed");
$usth->execute([$feed]); $usth->execute(["feed" => $feed]);
if ($tline = $usth->fetch()) { if ($tline = $usth->fetch()) {
Debug::log(sprintf("=> %s (ID: %d, UID: %d), last updated: %s", $tline["title"], $tline["id"], $tline["owner_uid"], Debug::log(sprintf("=> %s (ID: %d, UID: %d), last updated: %s", $tline["title"], $tline["id"], $tline["owner_uid"],
$tline["last_updated"] ? $tline["last_updated"] : "never")); $tline["last_updated"] ? $tline["last_updated"] : "never"));
if (array_search($tline["owner_uid"], $batch_owners) === false) if (!in_array($tline["owner_uid"], $batch_owners))
array_push($batch_owners, $tline["owner_uid"]); array_push($batch_owners, $tline["owner_uid"]);
$fstarted = microtime(true); $fstarted = microtime(true);