valvin
/
ttrss
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

update individual feed in a separate process to prevent PHP fatal errors 

(for example, OOM) from stopping the entire batch
this should also slightly increase memory budget for update processes
This commit is contained in:
Andrew Dolgov 2020-09-27 15:58:13 +03:00
parent e993d4feb2
commit 528b387563
2 changed files with 28 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
classes/rssutils.php
View File

@ -29,7 +29,7 @@ class RSSUtils {
$pdo->query("DELETE FROM ttrss_feedbrowser_cache"); $pdo->query("DELETE FROM ttrss_feedbrowser_cache");
} }
static function update_daemon_common($limit = DAEMON_FEED_LIMIT) { static function update_daemon_common($limit = DAEMON_FEED_LIMIT, $options = []) {
$schema_version = get_schema_version(); $schema_version = get_schema_version();
if ($schema_version != SCHEMA_VERSION) { if ($schema_version != SCHEMA_VERSION) {
@ -137,7 +137,6 @@ class RSSUtils {
Debug::log("Base feed: $feed"); Debug::log("Base feed: $feed");
$usth->execute([$feed]); $usth->execute([$feed]);
//update_rss_feed($line["id"], true);
if ($tline = $usth->fetch()) { if ($tline = $usth->fetch()) {
Debug::log(" => " . $tline["last_updated"] . ", " . $tline["id"] . " " . $tline["owner_uid"]); Debug::log(" => " . $tline["last_updated"] . ", " . $tline["id"] . " " . $tline["owner_uid"]);
@ -146,8 +145,15 @@ class RSSUtils {
array_push($batch_owners, $tline["owner_uid"]); array_push($batch_owners, $tline["owner_uid"]);
$fstarted = microtime(true); $fstarted = microtime(true);
$my_pid = posix_getpid();
try { $quiet = (isset($options["quiet"])) ? "--quiet" : "";
$log = function_exists("flock") && isset($options['log']) ? '--log '.$options['log'] : '';
$log_level = isset($options['log-level']) ? '--log-level '.$options['log-level'] : '';
passthru(PHP_EXECUTABLE . " update.php --update-feed " . $tline["id"] . " --pidlock $my_pid $quiet $log $log_level");
/* try {
self::update_rss_feed($tline["id"], true, false); self::update_rss_feed($tline["id"], true, false);
} catch (PDOException $e) { } catch (PDOException $e) {
Logger::get()->log_error(E_USER_NOTICE, $e->getMessage(), $e->getFile(), $e->getLine(), $e->getTraceAsString()); Logger::get()->log_error(E_USER_NOTICE, $e->getMessage(), $e->getFile(), $e->getLine(), $e->getTraceAsString());
@ -158,7 +164,7 @@ class RSSUtils {
// it doesn't matter if there wasn't actually anything to rollback, PDO Exception can be // it doesn't matter if there wasn't actually anything to rollback, PDO Exception can be
// thrown outside of an active transaction during feed update // thrown outside of an active transaction during feed update
} }
} } */
Debug::log(sprintf(" %.4f (sec)", microtime(true) - $fstarted)); Debug::log(sprintf(" %.4f (sec)", microtime(true) - $fstarted));

24
update.php
View File

@ -82,6 +82,7 @@
$longopts = array("feeds", $longopts = array("feeds",
"daemon", "daemon",
"daemon-loop", "daemon-loop",
"update-feed:",
"send-digests", "send-digests",
"task:", "task:",
"cleanup-tags", "cleanup-tags",
@ -235,8 +236,8 @@
} }
if (isset($options["feeds"])) { if (isset($options["feeds"])) {
RSSUtils::update_daemon_common(); RSSUtils::update_daemon_common(DAEMON_FEED_LIMIT, $options);
RSSUtils::housekeeping_common(true); RSSUtils::housekeeping_common();
PluginHost::getInstance()->run_hooks(PluginHost::HOOK_UPDATE_TASK, "hook_update_task", $op); PluginHost::getInstance()->run_hooks(PluginHost::HOOK_UPDATE_TASK, "hook_update_task", $op);
} }
@ -244,8 +245,8 @@
if (isset($options["daemon"])) { if (isset($options["daemon"])) {
while (true) { while (true) {
$quiet = (isset($options["quiet"])) ? "--quiet" : ""; $quiet = (isset($options["quiet"])) ? "--quiet" : "";
$log = isset($options['log']) ? '--log '.$options['log'] : ''; $log = isset($options['log']) ? '--log '.$options['log'] : '';
$log_level = isset($options['log-level']) ? '--log-level '.$options['log-level'] : ''; $log_level = isset($options['log-level']) ? '--log-level '.$options['log-level'] : '';
passthru(PHP_EXECUTABLE . " " . $argv[0] ." --daemon-loop $quiet $log $log_level"); passthru(PHP_EXECUTABLE . " " . $argv[0] ." --daemon-loop $quiet $log $log_level");
@ -257,15 +258,26 @@
} }
} }
if (isset($options["update-feed"])) {
try {
RSSUtils::update_rss_feed($options["update-feed"], true);
} catch (PDOException $e) {
Debug::log(sprintf("Exception while updating feed %d: %s (%s:%d)",
$options["update-feed"], $e->getMessage(), $e->getFile(), $e->getLine()));
Logger::get()->log_error(E_USER_NOTICE, $e->getMessage(), $e->getFile(), $e->getLine(), $e->getTraceAsString());
}
}
if (isset($options["daemon-loop"])) { if (isset($options["daemon-loop"])) {
if (!make_stampfile('update_daemon.stamp')) { if (!make_stampfile('update_daemon.stamp')) {
Debug::log("warning: unable to create stampfile\n"); Debug::log("warning: unable to create stampfile\n");
} }
RSSUtils::update_daemon_common(isset($options["pidlock"]) ? 50 : DAEMON_FEED_LIMIT); RSSUtils::update_daemon_common(isset($options["pidlock"]) ? 50 : DAEMON_FEED_LIMIT, $options);
if (!isset($options["pidlock"]) || $options["task"] == 0) if (!isset($options["pidlock"]) || $options["task"] == 0)
RSSUtils::housekeeping_common(true); RSSUtils::housekeeping_common();
PluginHost::getInstance()->run_hooks(PluginHost::HOOK_UPDATE_TASK, "hook_update_task", $op); PluginHost::getInstance()->run_hooks(PluginHost::HOOK_UPDATE_TASK, "hook_update_task", $op);
} }