From 6a40940ad6c6facea6c8e9d0dc1896885168c442 Mon Sep 17 00:00:00 2001 From: Andrew Dolgov Date: Fri, 2 May 2025 10:17:13 +0300 Subject: split housekeeping jobs to separate scheduled tasks on longer cooldown intervals, add table to record task execution timestamps, bump schema --- classes/Config.php | 2 +- classes/PluginHost.php | 90 +++++++++++++++++++++++++++++++++++++++++++++++ classes/RSSUtils.php | 95 ++++++++++++++++++++++++++++++++------------------ 3 files changed, 152 insertions(+), 35 deletions(-) (limited to 'classes') diff --git a/classes/Config.php b/classes/Config.php index c4176b7a8..1afa4f043 100644 --- a/classes/Config.php +++ b/classes/Config.php @@ -6,7 +6,7 @@ class Config { const T_STRING = 2; const T_INT = 3; - const SCHEMA_VERSION = 149; + const SCHEMA_VERSION = 150; /** override default values, defined below in _DEFAULTS[], prefixing with _ENVVAR_PREFIX: * diff --git a/classes/PluginHost.php b/classes/PluginHost.php index bfc02318b..484364f26 100644 --- a/classes/PluginHost.php +++ b/classes/PluginHost.php @@ -32,6 +32,9 @@ class PluginHost { /** @var array> */ private array $plugin_actions = []; + /** @var array */ + private array $scheduled_tasks = []; + private ?int $owner_uid = null; private bool $data_loaded = false; @@ -906,4 +909,91 @@ class PluginHost { $ref = new ReflectionClass(get_class($plugin)); return basename(dirname(dirname($ref->getFileName()))) == "plugins.local"; } + + /** + * Adds a backend scheduled task which will be executed by updater (if due) when idle during + * RSSUtils::housekeeping_common(). + * + * The granularity is not strictly guaranteed, housekeeping is invoked several times per hour + * depending on how fast feed batch was processed, but no more than once per minute. + * + * Tasks are not run in user context and are available to system plugins only. Task names may not + * overlap. + * + * Tasks should return an integer value (return code) which is stored in the database, a value of + * 0 is considered successful. + * + * @param string $task_name unique name for this task, plugins should prefix this with plugin name + * @param string $cron_expression schedule for this task in cron format + * @param Closure $callback task code that gets executed + */ + function add_scheduled_task(string $task_name, string $cron_expression, Closure $callback) : bool { + $task_name = strtolower($task_name); + + if (isset($this->scheduled_tasks[$task_name])) { + user_error("Attempted to override already registered scheduled task $task_name", E_USER_WARNING); + return false; + } else { + $cron = new Cron\CronExpression($cron_expression); + + $this->scheduled_tasks[$task_name] = [ + "cron" => $cron, + "callback" => $callback, + ]; + return true; + } + } + + /** + * Execute scheduled tasks which are due to run and record last run timestamps. + * @return void + */ + function run_due_tasks() { + Debug::log('Processing all scheduled tasks...'); + + $tasks_run = 0; + + foreach ($this->scheduled_tasks as $task_name => $task) { + $last_run = '1970-01-01 00:00'; + + $task_record = ORM::for_table('ttrss_scheduled_tasks') + ->where('task_name', $task_name) + ->find_one(); + + if ($task_record) + $last_run = $task_record->last_run; + + Debug::log("Checking scheduled task: $task_name, last run: $last_run"); + + if ($task['cron']->isDue($last_run)) { + Debug::log("Task $task_name is due, executing..."); + + $rc = (int) $task['callback'](); + + Debug::log("Task $task_name has finished with RC=$rc, recording timestamp..."); + + if ($task_record) { + $task_record->last_run = time(); + $task_record->last_rc = $rc; + + $task_record->save(); + } else { + $task_record = ORM::for_table('ttrss_scheduled_tasks')->create(); + + $task_record->set([ + 'task_name' => $task_name, + 'last_rc' => $rc, + 'last_run' => Db::NOW(), + ]); + + $task_record->save(); + } + } + } + + Debug::log("Finished with $tasks_run tasks executed."); + } + + // TODO implement some sort of automatic cleanup for orphan task execution records + } diff --git a/classes/RSSUtils.php b/classes/RSSUtils.php index acf1d14e5..d46d73793 100644 --- a/classes/RSSUtils.php +++ b/classes/RSSUtils.php @@ -35,11 +35,6 @@ class RSSUtils { return sha1(implode(",", $pluginhost->get_plugin_names()) . $tmp); } - static function cleanup_feed_browser(): void { - $pdo = Db::pdo(); - $pdo->query("DELETE FROM ttrss_feedbrowser_cache"); - } - static function cleanup_feed_icons(): void { $pdo = Db::pdo(); $sth = $pdo->prepare("SELECT id FROM ttrss_feeds WHERE id = ?"); @@ -81,6 +76,8 @@ class RSSUtils { die("Schema version is wrong, please upgrade the database.\n"); } + self::init_housekeeping_tasks(); + $pdo = Db::pdo(); $feeds_in_the_future = ORM::for_table('ttrss_feeds') @@ -1432,15 +1429,6 @@ class RSSUtils { WHERE ' . Db::past_comparison_qpart('created_at', '<', 7, 'day')); } - /** - * @deprecated table not used - */ - static function expire_feed_archive(): void { - $pdo = Db::pdo(); - - $pdo->query("DELETE FROM ttrss_archived_feeds"); - } - static function expire_lock_files(): void { Debug::log("Removing old lock files...", Debug::LOG_VERBOSE); @@ -1659,14 +1647,6 @@ class RSSUtils { mb_strtolower(strip_tags($title), 'utf-8')); } - /* counter cache is no longer used, if called truncate leftover data */ - static function cleanup_counters_cache(): void { - $pdo = Db::pdo(); - - $pdo->query("DELETE FROM ttrss_counters_cache"); - $pdo->query("DELETE FROM ttrss_cat_counters_cache"); - } - static function disable_failed_feeds(): void { if (Config::get(Config::DAEMON_UNSUCCESSFUL_DAYS_LIMIT) > 0) { @@ -1735,21 +1715,68 @@ class RSSUtils { } } - static function housekeeping_common(): void { - $cache = DiskCache::instance(""); - $cache->expire_all(); + /** Init all system tasks which are run periodically by updater in housekeeping_common() */ + static function init_housekeeping_tasks() : void { + Debug::log('Registering scheduled tasks for housekeeping...'); + + PluginHost::getInstance()->add_scheduled_task('purge_orphans', '@daily', + function() { + Article::_purge_orphans(); return 0; + } + ); + + PluginHost::getInstance()->add_scheduled_task('disk_cache_expire_all', '@daily', + function() { + $cache = DiskCache::instance(""); + $cache->expire_all(); + + return 0; + } + ); + + PluginHost::getInstance()->add_scheduled_task('expire_error_log', '@hourly', + function() { + self::expire_error_log(); + + return 0; + } + ); - self::migrate_feed_icons(); - self::expire_lock_files(); - self::expire_error_log(); - self::expire_feed_archive(); - self::cleanup_feed_browser(); - self::cleanup_feed_icons(); - self::disable_failed_feeds(); + PluginHost::getInstance()->add_scheduled_task('expire_lock_files', '@hourly', + function() { + self::expire_lock_files(); - Article::_purge_orphans(); - self::cleanup_counters_cache(); + return 0; + } + ); + + PluginHost::getInstance()->add_scheduled_task('disable_failed_feeds', '@daily', + function() { + self::disable_failed_feeds(); + + return 0; + } + ); + PluginHost::getInstance()->add_scheduled_task('migrate_feed_icons', '@daily', + function() { + self::migrate_feed_icons(); + + return 0; + } + ); + + PluginHost::getInstance()->add_scheduled_task('cleanup_feed_icons', '@daily', + function() { + self::cleanup_feed_icons(); + + return 0; + } + ); + } + + static function housekeeping_common(): void { + PluginHost::getInstance()->run_due_tasks(); PluginHost::getInstance()->run_hooks(PluginHost::HOOK_HOUSE_KEEPING); } -- cgit v1.2.3-54-g00ecf From a268f52de695fffb29769960332bfb34fe3ac7b5 Mon Sep 17 00:00:00 2001 From: Andrew Dolgov Date: Fri, 2 May 2025 10:23:30 +0300 Subject: record task duration in seconds --- classes/PluginHost.php | 6 +++++- sql/pgsql/migrations/150.sql | 1 + sql/pgsql/schema.sql | 1 + 3 files changed, 7 insertions(+), 1 deletion(-) (limited to 'classes') diff --git a/classes/PluginHost.php b/classes/PluginHost.php index 484364f26..e417fde1f 100644 --- a/classes/PluginHost.php +++ b/classes/PluginHost.php @@ -968,12 +968,15 @@ class PluginHost { if ($task['cron']->isDue($last_run)) { Debug::log("Task $task_name is due, executing..."); + $task_started = time(); $rc = (int) $task['callback'](); + $task_duration = time() - $task_started; - Debug::log("Task $task_name has finished with RC=$rc, recording timestamp..."); + Debug::log("Task $task_name has finished in $task_duration seconds with RC=$rc, recording timestamp..."); if ($task_record) { $task_record->last_run = time(); + $task_record->last_duration = $task_duration; $task_record->last_rc = $rc; $task_record->save(); @@ -982,6 +985,7 @@ class PluginHost { $task_record->set([ 'task_name' => $task_name, + 'last_duration' => $task_duration, 'last_rc' => $rc, 'last_run' => Db::NOW(), ]); diff --git a/sql/pgsql/migrations/150.sql b/sql/pgsql/migrations/150.sql index 55d9609e9..c0aae3bdf 100644 --- a/sql/pgsql/migrations/150.sql +++ b/sql/pgsql/migrations/150.sql @@ -1,5 +1,6 @@ create table ttrss_scheduled_tasks( id serial not null primary key, task_name varchar(250) unique not null, + last_duration integer not null, last_rc integer not null, last_run timestamp not null default NOW()); diff --git a/sql/pgsql/schema.sql b/sql/pgsql/schema.sql index 3ff9a6674..7e995f915 100644 --- a/sql/pgsql/schema.sql +++ b/sql/pgsql/schema.sql @@ -398,6 +398,7 @@ create table ttrss_error_log( create table ttrss_scheduled_tasks( id serial not null primary key, task_name varchar(250) unique not null, + last_duration integer not null, last_rc integer not null, last_run timestamp not null default NOW()); -- cgit v1.2.3-54-g00ecf From 44b5b33f3da9012e0028de6230ccbd5a729a4b71 Mon Sep 17 00:00:00 2001 From: Andrew Dolgov Date: Fri, 2 May 2025 10:28:35 +0300 Subject: remove synchronous usages of _purge_orphans() --- classes/RPC.php | 6 ------ classes/RSSUtils.php | 4 +++- 2 files changed, 3 insertions(+), 7 deletions(-) (limited to 'classes') diff --git a/classes/RPC.php b/classes/RPC.php index 6b6f3e909..c6cdab7f6 100644 --- a/classes/RPC.php +++ b/classes/RPC.php @@ -82,8 +82,6 @@ class RPC extends Handler_Protected { WHERE ref_id IN ($ids_qmarks) AND owner_uid = ?"); $sth->execute([...$ids, $_SESSION['uid']]); - Article::_purge_orphans(); - print json_encode(array("message" => "UPDATE_COUNTERS")); } @@ -311,10 +309,6 @@ class RPC extends Handler_Protected { } } - // Purge orphans and cleanup tags - Article::_purge_orphans(); - //cleanup_tags(14, 50000); - if ($num_updated > 0) { print json_encode(array("message" => "UPDATE_COUNTERS", "num_updated" => $num_updated)); diff --git a/classes/RSSUtils.php b/classes/RSSUtils.php index d46d73793..7c13dba63 100644 --- a/classes/RSSUtils.php +++ b/classes/RSSUtils.php @@ -1721,7 +1721,9 @@ class RSSUtils { PluginHost::getInstance()->add_scheduled_task('purge_orphans', '@daily', function() { - Article::_purge_orphans(); return 0; + Article::_purge_orphans(); + + return 0; } ); -- cgit v1.2.3-54-g00ecf From 36f60b51d7a49fe18071f67770064cccf2cb439d Mon Sep 17 00:00:00 2001 From: Andrew Dolgov Date: Fri, 2 May 2025 13:17:20 +0300 Subject: make digest sending a hourly cron job --- classes/RSSUtils.php | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) (limited to 'classes') diff --git a/classes/RSSUtils.php b/classes/RSSUtils.php index 7c13dba63..67ee3709d 100644 --- a/classes/RSSUtils.php +++ b/classes/RSSUtils.php @@ -285,9 +285,6 @@ class RSSUtils { self::housekeeping_user($owner_uid); } - // Send feed digests by email if needed. - Digest::send_headlines_digests(); - return $nf; } @@ -1775,6 +1772,14 @@ class RSSUtils { return 0; } ); + + PluginHost::getInstance()->add_scheduled_task('send_headlines_digests', '@hourly', + function() { + Digest::send_headlines_digests(); + + return 0; + } + ); } static function housekeeping_common(): void { -- cgit v1.2.3-54-g00ecf From a51c1d5176d0285a4fecc6e1e84d9f7dc4abaca5 Mon Sep 17 00:00:00 2001 From: Andrew Dolgov Date: Fri, 2 May 2025 13:18:48 +0300 Subject: fix tasks_run never incremented --- classes/PluginHost.php | 2 ++ 1 file changed, 2 insertions(+) (limited to 'classes') diff --git a/classes/PluginHost.php b/classes/PluginHost.php index e417fde1f..9bdf7c0b4 100644 --- a/classes/PluginHost.php +++ b/classes/PluginHost.php @@ -972,6 +972,8 @@ class PluginHost { $rc = (int) $task['callback'](); $task_duration = time() - $task_started; + ++$tasks_run; + Debug::log("Task $task_name has finished in $task_duration seconds with RC=$rc, recording timestamp..."); if ($task_record) { -- cgit v1.2.3-54-g00ecf From aeca30cb0c6b26c569f66c5043690d5528fc481b Mon Sep 17 00:00:00 2001 From: Andrew Dolgov Date: Fri, 2 May 2025 13:26:58 +0300 Subject: drop SIMPLE_UPDATE_MODE, limit housekeeping and updates to background processes --- backend.php | 2 +- classes/Config.php | 8 ------ classes/Handler_Public.php | 14 --------- classes/RPC.php | 72 ---------------------------------------------- classes/RSSUtils.php | 1 - js/App.js | 5 ---- js/Feeds.js | 7 ----- 7 files changed, 1 insertion(+), 108 deletions(-) (limited to 'classes') diff --git a/backend.php b/backend.php index 860a3940c..d9a0af7c7 100644 --- a/backend.php +++ b/backend.php @@ -14,7 +14,7 @@ /* Public calls compatibility shim */ - $public_calls = array("globalUpdateFeeds", "rss", "getUnread", "getProfiles", "share"); + $public_calls = array("rss", "getUnread", "getProfiles", "share"); if (array_search($op, $public_calls) !== false) { header("Location: public.php?" . $_SERVER['QUERY_STRING']); diff --git a/classes/Config.php b/classes/Config.php index 1afa4f043..c413317be 100644 --- a/classes/Config.php +++ b/classes/Config.php @@ -53,13 +53,6 @@ class Config { * your tt-rss directory protected by other means (e.g. http auth). */ const SINGLE_USER_MODE = "SINGLE_USER_MODE"; - /** enables fallback update mode where tt-rss tries to update feeds in - * background while tt-rss is open in your browser. - * if you don't have a lot of feeds and don't want to or can't run - * background processes while not running tt-rss, this method is generally - * viable to keep your feeds up to date. */ - const SIMPLE_UPDATE_MODE = "SIMPLE_UPDATE_MODE"; - /** use this PHP CLI executable to start various tasks */ const PHP_EXECUTABLE = "PHP_EXECUTABLE"; @@ -205,7 +198,6 @@ class Config { Config::DB_PORT => [ "5432", Config::T_STRING ], Config::SELF_URL_PATH => [ "https://example.com/tt-rss", Config::T_STRING ], Config::SINGLE_USER_MODE => [ "", Config::T_BOOL ], - Config::SIMPLE_UPDATE_MODE => [ "", Config::T_BOOL ], Config::PHP_EXECUTABLE => [ "/usr/bin/php", Config::T_STRING ], Config::LOCK_DIRECTORY => [ "lock", Config::T_STRING ], Config::CACHE_DIR => [ "cache", Config::T_STRING ], diff --git a/classes/Handler_Public.php b/classes/Handler_Public.php index abff08376..bb3667e2a 100644 --- a/classes/Handler_Public.php +++ b/classes/Handler_Public.php @@ -360,20 +360,6 @@ class Handler_Public extends Handler { header('HTTP/1.1 403 Forbidden'); } - function updateTask(): void { - PluginHost::getInstance()->run_hooks(PluginHost::HOOK_UPDATE_TASK); - } - - function housekeepingTask(): void { - PluginHost::getInstance()->run_hooks(PluginHost::HOOK_HOUSE_KEEPING); - } - - function globalUpdateFeeds(): void { - RPC::updaterandomfeed_real(); - - PluginHost::getInstance()->run_hooks(PluginHost::HOOK_UPDATE_TASK); - } - function login(): void { if (!Config::get(Config::SINGLE_USER_MODE)) { diff --git a/classes/RPC.php b/classes/RPC.php index c6cdab7f6..4d133c272 100644 --- a/classes/RPC.php +++ b/classes/RPC.php @@ -250,77 +250,6 @@ class RPC extends Handler_Protected { print json_encode(["wide" => $wide]); } - static function updaterandomfeed_real(): void { - $default_interval = (int) Prefs::get_default(Prefs::DEFAULT_UPDATE_INTERVAL); - - // Test if the feed need a update (update interval exceded). - $update_limit_qpart = "AND (( - update_interval = 0 - AND (p.value IS NULL OR p.value != '-1') - AND last_updated < NOW() - CAST((COALESCE(p.value, '$default_interval') || ' minutes') AS INTERVAL) - ) OR ( - update_interval > 0 - AND last_updated < NOW() - CAST((update_interval || ' minutes') AS INTERVAL) - ) OR ( - update_interval >= 0 - AND (p.value IS NULL OR p.value != '-1') - AND (last_updated = '1970-01-01 00:00:00' OR last_updated IS NULL) - ))"; - - // Test if feed is currently being updated by another process. - $updstart_thresh_qpart = 'AND (last_update_started IS NULL OR ' - . Db::past_comparison_qpart('last_update_started', '<', 5, 'minute') . ')'; - - $pdo = Db::pdo(); - - // we could be invoked from public.php with no active session - if (!empty($_SESSION["uid"])) { - $owner_check_qpart = "AND f.owner_uid = ".$pdo->quote($_SESSION["uid"]); - } else { - $owner_check_qpart = ""; - } - - $query = "SELECT f.feed_url,f.id - FROM - ttrss_feeds f, ttrss_users u LEFT JOIN ttrss_user_prefs2 p ON - (p.owner_uid = u.id AND profile IS NULL AND pref_name = 'DEFAULT_UPDATE_INTERVAL') - WHERE - f.owner_uid = u.id AND - u.access_level NOT IN (".sprintf("%d, %d", UserHelper::ACCESS_LEVEL_DISABLED, UserHelper::ACCESS_LEVEL_READONLY).") - $owner_check_qpart - $update_limit_qpart - $updstart_thresh_qpart - ORDER BY RANDOM() LIMIT 30"; - - $res = $pdo->query($query); - - $num_updated = 0; - - $tstart = time(); - - while ($line = $res->fetch()) { - $feed_id = $line["id"]; - - if (time() - $tstart < ini_get("max_execution_time") * 0.7) { - RSSUtils::update_rss_feed($feed_id, true); - ++$num_updated; - } else { - break; - } - } - - if ($num_updated > 0) { - print json_encode(array("message" => "UPDATE_COUNTERS", - "num_updated" => $num_updated)); - } else { - print json_encode(array("message" => "NOTHING_TO_UPDATE")); - } - } - - function updaterandomfeed(): void { - self::updaterandomfeed_real(); - } - /** * @param array $ids */ @@ -466,7 +395,6 @@ class RPC extends Handler_Protected { $params["num_feeds"] = (int) $num_feeds; $params["hotkeys"] = $this->get_hotkeys_map(); $params["widescreen"] = (int) Prefs::get(Prefs::WIDESCREEN_MODE, $_SESSION['uid'], $profile); - $params['simple_update'] = Config::get(Config::SIMPLE_UPDATE_MODE); $params["icon_indicator_white"] = $this->image_to_base64("images/indicator_white.gif"); $params["icon_oval"] = $this->image_to_base64("images/oval.svg"); $params["icon_three_dots"] = $this->image_to_base64("images/three-dots.svg"); diff --git a/classes/RSSUtils.php b/classes/RSSUtils.php index 67ee3709d..eef0d8540 100644 --- a/classes/RSSUtils.php +++ b/classes/RSSUtils.php @@ -129,7 +129,6 @@ class RSSUtils { ))"; // Test if feed is currently being updated by another process. - // TODO: Update RPC::updaterandomfeed_real() to also use 10 minutes? $updstart_thresh_qpart = 'AND (last_update_started IS NULL OR ' . Db::past_comparison_qpart('last_update_started', '<', 10, 'minute') . ')'; diff --git a/js/App.js b/js/App.js index 0c4c72f22..f1c4bc1c3 100644 --- a/js/App.js +++ b/js/App.js @@ -829,11 +829,6 @@ const App = { Headlines.initScrollHandler(); - if (this.getInitParam("simple_update")) { - console.log("scheduling simple feed updater..."); - window.setInterval(() => { Feeds.updateRandom() }, 30 * 1000); - } - if (this.getInitParam('check_for_updates')) { window.setInterval(() => { this.checkForUpdates(); diff --git a/js/Feeds.js b/js/Feeds.js index 9088b7efc..7a58a10a4 100644 --- a/js/Feeds.js +++ b/js/Feeds.js @@ -743,13 +743,6 @@ const Feeds = { dialog.show(); }, - updateRandom: function() { - console.log("in update_random_feed"); - - xhr.json("backend.php", {op: "RPC", method: "updaterandomfeed"}, () => { - // - }); - }, renderIcon: function(feed_id, exists) { const icon_url = App.getInitParam("icons_url") + '?' + dojo.objectToQuery({op: 'feed_icon', id: feed_id}); -- cgit v1.2.3-54-g00ecf From 247efe3137fadf5d74ab254cf4c80957624abc90 Mon Sep 17 00:00:00 2001 From: Andrew Dolgov Date: Fri, 2 May 2025 13:37:08 +0300 Subject: bring back cleanup of potentially sensitive environment variables but exclude CLI SAPI to prevent updater failures --- classes/Config.php | 14 ++++++++++++++ 1 file changed, 14 insertions(+) (limited to 'classes') diff --git a/classes/Config.php b/classes/Config.php index c413317be..6e16f269b 100644 --- a/classes/Config.php +++ b/classes/Config.php @@ -275,6 +275,13 @@ class Config { if (isset(self::_DEFAULTS[$const])) { $override = getenv(self::_ENVVAR_PREFIX . $const); + // cleanup original env var after importing (unless it's a background process) + if (php_sapi_name() != "cli") { + putenv(self::_ENVVAR_PREFIX . $const . '='); + unset($_ENV[self::_ENVVAR_PREFIX . $const]); + unset($_SERVER[self::_ENVVAR_PREFIX . $const]); + } + list ($defval, $deftype) = self::_DEFAULTS[$const]; $this->params[$cvalue] = [ self::cast_to($override !== false ? $override : $defval, $deftype), $deftype ]; @@ -433,6 +440,13 @@ class Config { private function _add(string $param, string $default, int $type_hint): void { $override = getenv(self::_ENVVAR_PREFIX . $param); + // cleanup original env var after importing (unless it's a background process) + if (php_sapi_name() != "cli") { + putenv(self::_ENVVAR_PREFIX . $param . '='); + unset($_ENV[self::_ENVVAR_PREFIX . $param]); + unset($_SERVER[self::_ENVVAR_PREFIX . $param]); + } + $this->params[$param] = [ self::cast_to($override !== false ? $override : $default, $type_hint), $type_hint ]; } -- cgit v1.2.3-54-g00ecf From dc6ea08ca490c889f4e85bd697e6bdffb95a22f4 Mon Sep 17 00:00:00 2001 From: Andrew Dolgov Date: Fri, 2 May 2025 14:03:45 +0300 Subject: add workaround for due tasks because housekeeping is not run every minute, fix last_run not updated to NOW() in the db --- classes/PluginHost.php | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'classes') diff --git a/classes/PluginHost.php b/classes/PluginHost.php index 9bdf7c0b4..8bde8df3f 100644 --- a/classes/PluginHost.php +++ b/classes/PluginHost.php @@ -965,7 +965,9 @@ class PluginHost { Debug::log("Checking scheduled task: $task_name, last run: $last_run"); - if ($task['cron']->isDue($last_run)) { + // because we don't schedule tasks every minute, we assume that task is due if its + // next estimated run based on previous timestamp is in the past + if ($task['cron']->getNextRunDate($last_run)->getTimestamp() - time() < 0) { Debug::log("Task $task_name is due, executing..."); $task_started = time(); @@ -977,7 +979,7 @@ class PluginHost { Debug::log("Task $task_name has finished in $task_duration seconds with RC=$rc, recording timestamp..."); if ($task_record) { - $task_record->last_run = time(); + $task_record->last_run = Db::NOW(); $task_record->last_duration = $task_duration; $task_record->last_rc = $rc; -- cgit v1.2.3-54-g00ecf From b30f8c93a00ce1ae2c582ca4c7f1d5d8425220ee Mon Sep 17 00:00:00 2001 From: Andrew Dolgov Date: Fri, 2 May 2025 21:27:50 +0300 Subject: rename article mark/publish hooks --- classes/API.php | 4 ++-- classes/Article.php | 6 +++--- classes/Plugin.php | 7 +++++-- classes/PluginHost.php | 8 ++++---- classes/RPC.php | 8 ++++---- classes/RSSUtils.php | 4 ++-- 6 files changed, 20 insertions(+), 17 deletions(-) (limited to 'classes') diff --git a/classes/API.php b/classes/API.php index 83eaa22b8..6db05198a 100644 --- a/classes/API.php +++ b/classes/API.php @@ -285,10 +285,10 @@ class API extends Handler { $sth->execute([...$article_ids, $_SESSION['uid']]); if ($field == 'marked') - PluginHost::getInstance()->run_hooks(PluginHost::HOOK_ARTICLES_MARKED, $article_ids); + PluginHost::getInstance()->run_hooks(PluginHost::HOOK_ARTICLES_MARK_TOGGLED, $article_ids); if ($field == 'published') - PluginHost::getInstance()->run_hooks(PluginHost::HOOK_ARTICLES_PUBLISHED, $article_ids); + PluginHost::getInstance()->run_hooks(PluginHost::HOOK_ARTICLES_PUBLISH_TOGGLED, $article_ids); $num_updated = $sth->rowCount(); diff --git a/classes/Article.php b/classes/Article.php index 6a3111892..c0d77123c 100644 --- a/classes/Article.php +++ b/classes/Article.php @@ -98,7 +98,7 @@ class Article extends Handler_Protected { int_id = ? AND owner_uid = ?"); $sth->execute([$int_id, $owner_uid]); - PluginHost::getInstance()->run_hooks(PluginHost::HOOK_ARTICLES_PUBLISHED, [$ref_id]); + PluginHost::getInstance()->run_hooks(PluginHost::HOOK_ARTICLES_PUBLISH_TOGGLED, [$ref_id]); } else { @@ -109,7 +109,7 @@ class Article extends Handler_Protected { (?, '', NULL, NULL, ?, true, '', '', NOW(), '', false, NOW())"); $sth->execute([$ref_id, $owner_uid]); - PluginHost::getInstance()->run_hooks(PluginHost::HOOK_ARTICLES_PUBLISHED, [$ref_id]); + PluginHost::getInstance()->run_hooks(PluginHost::HOOK_ARTICLES_PUBLISH_TOGGLED, [$ref_id]); } if (count($labels) != 0) { @@ -148,7 +148,7 @@ class Article extends Handler_Protected { (?, '', NULL, NULL, ?, true, '', '', NOW(), '', false, NOW())"); $sth->execute([$ref_id, $owner_uid]); - PluginHost::getInstance()->run_hooks(PluginHost::HOOK_ARTICLES_PUBLISHED, [$ref_id]); + PluginHost::getInstance()->run_hooks(PluginHost::HOOK_ARTICLES_PUBLISH_TOGGLED, [$ref_id]); if (count($labels) != 0) { foreach ($labels as $label) { diff --git a/classes/Plugin.php b/classes/Plugin.php index b66e2082c..3165d62f4 100644 --- a/classes/Plugin.php +++ b/classes/Plugin.php @@ -714,22 +714,25 @@ abstract class Plugin { } /** Invoked after passed article IDs were either marked (i.e. starred) or unmarked. + * * **Note** resulting state of the articles is not passed to this function (because * tt-rss may do invert operation on ID range), you will need to get this from the database. * @param array $article_ids ref_ids * @return void */ - function hook_articles_marked(array $article_ids) { + function hook_articles_mark_toggled(array $article_ids) { user_error("Dummy method invoked.", E_USER_ERROR); } /** Invoked after passed article IDs were either published or unpublished. + * * **Note** resulting state of the articles is not passed to this function (because * tt-rss may do invert operation on ID range), you will need to get this from the database. + * * @param array $article_ids ref_ids * @return void */ - function hook_articles_published(array $article_ids) { + function hook_articles_publish_toggled(array $article_ids) { user_error("Dummy method invoked.", E_USER_ERROR); } } diff --git a/classes/PluginHost.php b/classes/PluginHost.php index 8bde8df3f..5cff4afcb 100644 --- a/classes/PluginHost.php +++ b/classes/PluginHost.php @@ -202,11 +202,11 @@ class PluginHost { /** @see Plugin::hook_validate_session() */ const HOOK_VALIDATE_SESSION = "hook_validate_session"; - /** @see Plugin::hook_articles_marked() */ - const HOOK_ARTICLES_MARKED = "hook_articles_marked"; + /** @see Plugin::hook_articles_mark_toggled() */ + const HOOK_ARTICLES_MARK_TOGGLED = "hook_articles_mark_toggled"; - /** @see Plugin::hook_articles_published() */ - const HOOK_ARTICLES_PUBLISHED = "hook_articles_published"; + /** @see Plugin::hook_articles_publish_toggled() */ + const HOOK_ARTICLES_PUBLISH_TOGGLED = "hook_articles_publish_toggled"; const KIND_ALL = 1; const KIND_SYSTEM = 2; diff --git a/classes/RPC.php b/classes/RPC.php index 4d133c272..6ce2c12aa 100644 --- a/classes/RPC.php +++ b/classes/RPC.php @@ -69,7 +69,7 @@ class RPC extends Handler_Protected { $sth->execute([$mark, $id, $_SESSION['uid']]); - PluginHost::getInstance()->run_hooks(PluginHost::HOOK_ARTICLES_MARKED, [$id]); + PluginHost::getInstance()->run_hooks(PluginHost::HOOK_ARTICLES_MARK_TOGGLED, [$id]); print json_encode(array("message" => "UPDATE_COUNTERS")); } @@ -95,7 +95,7 @@ class RPC extends Handler_Protected { $sth->execute([$pub, $id, $_SESSION['uid']]); - PluginHost::getInstance()->run_hooks(PluginHost::HOOK_ARTICLES_PUBLISHED, [$id]); + PluginHost::getInstance()->run_hooks(PluginHost::HOOK_ARTICLES_PUBLISH_TOGGLED, [$id]); print json_encode(array("message" => "UPDATE_COUNTERS")); } @@ -273,7 +273,7 @@ class RPC extends Handler_Protected { $sth->execute([...$ids, $_SESSION['uid']]); - PluginHost::getInstance()->run_hooks(PluginHost::HOOK_ARTICLES_MARKED, $ids); + PluginHost::getInstance()->run_hooks(PluginHost::HOOK_ARTICLES_MARK_TOGGLED, $ids); } /** @@ -299,7 +299,7 @@ class RPC extends Handler_Protected { $sth->execute([...$ids, $_SESSION['uid']]); - PluginHost::getInstance()->run_hooks(PluginHost::HOOK_ARTICLES_PUBLISHED, $ids); + PluginHost::getInstance()->run_hooks(PluginHost::HOOK_ARTICLES_PUBLISH_TOGGLED, $ids); } function log(): void { diff --git a/classes/RSSUtils.php b/classes/RSSUtils.php index eef0d8540..575a1eda1 100644 --- a/classes/RSSUtils.php +++ b/classes/RSSUtils.php @@ -1126,10 +1126,10 @@ class RSSUtils { $published, $score]); if ($marked) - PluginHost::getInstance()->run_hooks(PluginHost::HOOK_ARTICLES_MARKED, [$ref_id]); + PluginHost::getInstance()->run_hooks(PluginHost::HOOK_ARTICLES_MARK_TOGGLED, [$ref_id]); if ($published) - PluginHost::getInstance()->run_hooks(PluginHost::HOOK_ARTICLES_PUBLISHED, [$ref_id]); + PluginHost::getInstance()->run_hooks(PluginHost::HOOK_ARTICLES_PUBLISH_TOGGLED, [$ref_id]); $sth = $pdo->prepare("SELECT int_id FROM ttrss_user_entries WHERE ref_id = ? AND owner_uid = ? AND -- cgit v1.2.3-54-g00ecf From d5d15072e193ba96f156fea3e32bcb0af96b7b63 Mon Sep 17 00:00:00 2001 From: Andrew Dolgov Date: Fri, 2 May 2025 22:51:07 +0300 Subject: move scheduled tasks to a separate class, add some try-catches, improve/shorten logging and descriptions --- classes/PluginHost.php | 97 -------------------------------------- classes/RSSUtils.php | 21 +++++---- classes/Scheduler.php | 124 +++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 136 insertions(+), 106 deletions(-) create mode 100644 classes/Scheduler.php (limited to 'classes') diff --git a/classes/PluginHost.php b/classes/PluginHost.php index 5cff4afcb..f61a5a9a4 100644 --- a/classes/PluginHost.php +++ b/classes/PluginHost.php @@ -32,9 +32,6 @@ class PluginHost { /** @var array> */ private array $plugin_actions = []; - /** @var array */ - private array $scheduled_tasks = []; - private ?int $owner_uid = null; private bool $data_loaded = false; @@ -910,98 +907,4 @@ class PluginHost { return basename(dirname(dirname($ref->getFileName()))) == "plugins.local"; } - /** - * Adds a backend scheduled task which will be executed by updater (if due) when idle during - * RSSUtils::housekeeping_common(). - * - * The granularity is not strictly guaranteed, housekeeping is invoked several times per hour - * depending on how fast feed batch was processed, but no more than once per minute. - * - * Tasks are not run in user context and are available to system plugins only. Task names may not - * overlap. - * - * Tasks should return an integer value (return code) which is stored in the database, a value of - * 0 is considered successful. - * - * @param string $task_name unique name for this task, plugins should prefix this with plugin name - * @param string $cron_expression schedule for this task in cron format - * @param Closure $callback task code that gets executed - */ - function add_scheduled_task(string $task_name, string $cron_expression, Closure $callback) : bool { - $task_name = strtolower($task_name); - - if (isset($this->scheduled_tasks[$task_name])) { - user_error("Attempted to override already registered scheduled task $task_name", E_USER_WARNING); - return false; - } else { - $cron = new Cron\CronExpression($cron_expression); - - $this->scheduled_tasks[$task_name] = [ - "cron" => $cron, - "callback" => $callback, - ]; - return true; - } - } - - /** - * Execute scheduled tasks which are due to run and record last run timestamps. - * @return void - */ - function run_due_tasks() { - Debug::log('Processing all scheduled tasks...'); - - $tasks_run = 0; - - foreach ($this->scheduled_tasks as $task_name => $task) { - $last_run = '1970-01-01 00:00'; - - $task_record = ORM::for_table('ttrss_scheduled_tasks') - ->where('task_name', $task_name) - ->find_one(); - - if ($task_record) - $last_run = $task_record->last_run; - - Debug::log("Checking scheduled task: $task_name, last run: $last_run"); - - // because we don't schedule tasks every minute, we assume that task is due if its - // next estimated run based on previous timestamp is in the past - if ($task['cron']->getNextRunDate($last_run)->getTimestamp() - time() < 0) { - Debug::log("Task $task_name is due, executing..."); - - $task_started = time(); - $rc = (int) $task['callback'](); - $task_duration = time() - $task_started; - - ++$tasks_run; - - Debug::log("Task $task_name has finished in $task_duration seconds with RC=$rc, recording timestamp..."); - - if ($task_record) { - $task_record->last_run = Db::NOW(); - $task_record->last_duration = $task_duration; - $task_record->last_rc = $rc; - - $task_record->save(); - } else { - $task_record = ORM::for_table('ttrss_scheduled_tasks')->create(); - - $task_record->set([ - 'task_name' => $task_name, - 'last_duration' => $task_duration, - 'last_rc' => $rc, - 'last_run' => Db::NOW(), - ]); - - $task_record->save(); - } - } - } - - Debug::log("Finished with $tasks_run tasks executed."); - } - - // TODO implement some sort of automatic cleanup for orphan task execution records - } diff --git a/classes/RSSUtils.php b/classes/RSSUtils.php index 575a1eda1..bf05e0b98 100644 --- a/classes/RSSUtils.php +++ b/classes/RSSUtils.php @@ -1715,7 +1715,9 @@ class RSSUtils { static function init_housekeeping_tasks() : void { Debug::log('Registering scheduled tasks for housekeeping...'); - PluginHost::getInstance()->add_scheduled_task('purge_orphans', '@daily', + $scheduler = Scheduler::getInstance(); + + $scheduler->add_scheduled_task('purge_orphans', '@daily', function() { Article::_purge_orphans(); @@ -1723,7 +1725,7 @@ class RSSUtils { } ); - PluginHost::getInstance()->add_scheduled_task('disk_cache_expire_all', '@daily', + $scheduler->add_scheduled_task('disk_cache_expire_all', '@daily', function() { $cache = DiskCache::instance(""); $cache->expire_all(); @@ -1732,7 +1734,7 @@ class RSSUtils { } ); - PluginHost::getInstance()->add_scheduled_task('expire_error_log', '@hourly', + $scheduler->add_scheduled_task('expire_error_log', '@hourly', function() { self::expire_error_log(); @@ -1740,7 +1742,7 @@ class RSSUtils { } ); - PluginHost::getInstance()->add_scheduled_task('expire_lock_files', '@hourly', + $scheduler->add_scheduled_task('expire_lock_files', '@hourly', function() { self::expire_lock_files(); @@ -1748,7 +1750,7 @@ class RSSUtils { } ); - PluginHost::getInstance()->add_scheduled_task('disable_failed_feeds', '@daily', + $scheduler->add_scheduled_task('disable_failed_feeds', '@daily', function() { self::disable_failed_feeds(); @@ -1756,7 +1758,7 @@ class RSSUtils { } ); - PluginHost::getInstance()->add_scheduled_task('migrate_feed_icons', '@daily', + $scheduler->add_scheduled_task('migrate_feed_icons', '@daily', function() { self::migrate_feed_icons(); @@ -1764,7 +1766,7 @@ class RSSUtils { } ); - PluginHost::getInstance()->add_scheduled_task('cleanup_feed_icons', '@daily', + $scheduler->add_scheduled_task('cleanup_feed_icons', '@daily', function() { self::cleanup_feed_icons(); @@ -1772,7 +1774,7 @@ class RSSUtils { } ); - PluginHost::getInstance()->add_scheduled_task('send_headlines_digests', '@hourly', + $scheduler->add_scheduled_task('send_headlines_digests', '@hourly', function() { Digest::send_headlines_digests(); @@ -1782,7 +1784,8 @@ class RSSUtils { } static function housekeeping_common(): void { - PluginHost::getInstance()->run_due_tasks(); + Scheduler::getInstance()->run_due_tasks(); + PluginHost::getInstance()->run_hooks(PluginHost::HOOK_HOUSE_KEEPING); } diff --git a/classes/Scheduler.php b/classes/Scheduler.php new file mode 100644 index 000000000..f0ca38303 --- /dev/null +++ b/classes/Scheduler.php @@ -0,0 +1,124 @@ + */ + private array $scheduled_tasks = []; + + public static function getInstance(): Scheduler { + if (self::$instance == null) + self::$instance = new self(); + + return self::$instance; + } + + /** + * Adds a backend scheduled task which will be executed by updater (if due) during housekeeping. + * + * The granularity is not strictly guaranteed, housekeeping is invoked several times per hour + * depending on how fast feed batch was processed, but no more than once per minute. + * + * Tasks do not run in user context. Task names may not overlap. Plugins should register tasks + * via PluginHost methods (to be implemented later). + * + * Tasks should return an integer value (return code) which is stored in the database, a value of + * 0 is considered successful. + * + * @param string $task_name unique name for this task, plugins should prefix this with plugin name + * @param string $cron_expression schedule for this task in cron format + * @param Closure $callback task code that gets executed + */ + function add_scheduled_task(string $task_name, string $cron_expression, Closure $callback) : bool { + $task_name = strtolower($task_name); + + if (isset($this->scheduled_tasks[$task_name])) { + user_error("Attempted to override already registered scheduled task $task_name", E_USER_WARNING); + return false; + } else { + try { + $cron = new Cron\CronExpression($cron_expression); + } catch (InvalidArgumentException $e) { + user_error("Attempt to register scheduled task $task_name failed: " . $e->getMessage(), E_USER_WARNING); + return false; + } + + $this->scheduled_tasks[$task_name] = [ + "cron" => $cron, + "callback" => $callback, + ]; + return true; + } + } + + /** + * Execute scheduled tasks which are due to run and record last run timestamps. + */ + function run_due_tasks() : void { + Debug::log('Processing all scheduled tasks...'); + + $tasks_succeeded = 0; + $tasks_failed = 0; + + foreach ($this->scheduled_tasks as $task_name => $task) { + $task_record = ORM::for_table('ttrss_scheduled_tasks') + ->where('task_name', $task_name) + ->find_one(); + + if ($task_record) + $last_run = $task_record->last_run; + else + $last_run = '1970-01-01 00:00'; + + // because we don't schedule tasks every minute, we assume that task is due if its + // next estimated run based on previous timestamp is in the past + if ($task['cron']->getNextRunDate($last_run)->getTimestamp() - time() < 0) { + Debug::log("Task $task_name is due, executing..."); + + $task_started = time(); + + try { + $rc = (int) $task['callback'](); + } catch (Exception $e) { + user_error("Scheduled task $task_name failed with exception: " . $e->getMessage(), E_USER_WARNING); + + $rc = self::TASK_RC_EXCEPTION; + } + + $task_duration = time() - $task_started; + + if ($rc === 0) { + ++$tasks_succeeded; + Debug::log("Task $task_name has finished in $task_duration seconds."); + } else { + $tasks_failed++; + Debug::log("Task $task_name has failed with RC: $rc after $task_duration seconds."); + } + + if ($task_record) { + $task_record->last_run = Db::NOW(); + $task_record->last_duration = $task_duration; + $task_record->last_rc = $rc; + + $task_record->save(); + } else { + $task_record = ORM::for_table('ttrss_scheduled_tasks')->create(); + + $task_record->set([ + 'task_name' => $task_name, + 'last_duration' => $task_duration, + 'last_rc' => $rc, + 'last_run' => Db::NOW(), + ]); + + $task_record->save(); + } + } + } + + Debug::log("Finished with $tasks_succeeded tasks succeeded and $tasks_failed tasks failed."); + } + + // TODO implement some sort of automatic cleanup for orphan task execution records +} \ No newline at end of file -- cgit v1.2.3-54-g00ecf From 997c10437e23552285e450f389e8a84ec4b04f5e Mon Sep 17 00:00:00 2001 From: Andrew Dolgov Date: Fri, 2 May 2025 23:26:13 +0300 Subject: reorder housekeeping tasks by interval --- classes/RSSUtils.php | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) (limited to 'classes') diff --git a/classes/RSSUtils.php b/classes/RSSUtils.php index bf05e0b98..a051a7dc2 100644 --- a/classes/RSSUtils.php +++ b/classes/RSSUtils.php @@ -1734,41 +1734,41 @@ class RSSUtils { } ); - $scheduler->add_scheduled_task('expire_error_log', '@hourly', + $scheduler->add_scheduled_task('disable_failed_feeds', '@daily', function() { - self::expire_error_log(); + self::disable_failed_feeds(); return 0; } ); - $scheduler->add_scheduled_task('expire_lock_files', '@hourly', + $scheduler->add_scheduled_task('migrate_feed_icons', '@daily', function() { - self::expire_lock_files(); + self::migrate_feed_icons(); return 0; } ); - $scheduler->add_scheduled_task('disable_failed_feeds', '@daily', + $scheduler->add_scheduled_task('cleanup_feed_icons', '@daily', function() { - self::disable_failed_feeds(); + self::cleanup_feed_icons(); return 0; } ); - $scheduler->add_scheduled_task('migrate_feed_icons', '@daily', + $scheduler->add_scheduled_task('expire_error_log', '@hourly', function() { - self::migrate_feed_icons(); + self::expire_error_log(); return 0; } ); - $scheduler->add_scheduled_task('cleanup_feed_icons', '@daily', + $scheduler->add_scheduled_task('expire_lock_files', '@hourly', function() { - self::cleanup_feed_icons(); + self::expire_lock_files(); return 0; } -- cgit v1.2.3-54-g00ecf From 4cda1da5c0c511fb2938d8b3683cbef1d75377da Mon Sep 17 00:00:00 2001 From: Andrew Dolgov Date: Sat, 3 May 2025 07:55:16 +0300 Subject: adjust scheduler logging to be somewhat more alike to feed updater --- classes/Scheduler.php | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'classes') diff --git a/classes/Scheduler.php b/classes/Scheduler.php index f0ca38303..fd6301641 100644 --- a/classes/Scheduler.php +++ b/classes/Scheduler.php @@ -74,7 +74,7 @@ class Scheduler { // because we don't schedule tasks every minute, we assume that task is due if its // next estimated run based on previous timestamp is in the past if ($task['cron']->getNextRunDate($last_run)->getTimestamp() - time() < 0) { - Debug::log("Task $task_name is due, executing..."); + Debug::log("=> Scheduled task $task_name is due, executing..."); $task_started = time(); @@ -90,10 +90,10 @@ class Scheduler { if ($rc === 0) { ++$tasks_succeeded; - Debug::log("Task $task_name has finished in $task_duration seconds."); + Debug::log("<= Scheduled task $task_name has finished in $task_duration seconds."); } else { $tasks_failed++; - Debug::log("Task $task_name has failed with RC: $rc after $task_duration seconds."); + Debug::log("!! Scheduled task $task_name has failed with RC: $rc after $task_duration seconds."); } if ($task_record) { @@ -117,7 +117,7 @@ class Scheduler { } } - Debug::log("Finished with $tasks_succeeded tasks succeeded and $tasks_failed tasks failed."); + Debug::log("Processing scheduled tasks finished with $tasks_succeeded tasks succeeded and $tasks_failed tasks failed."); } // TODO implement some sort of automatic cleanup for orphan task execution records -- cgit v1.2.3-54-g00ecf