2012-06-20 05:35:34 +00:00
#!/usr/bin/env php
2007-02-24 16:46:39 +00:00
< ? php
2012-12-09 09:41:22 +00:00
set_include_path ( dirname ( __FILE__ ) . " /include " . PATH_SEPARATOR .
get_include_path ());
2011-12-11 19:59:25 +00:00
2010-11-09 13:33:08 +00:00
define ( 'DISABLE_SESSIONS' , true );
2011-04-15 08:38:11 +00:00
chdir ( dirname ( __FILE__ ));
2013-04-17 11:36:34 +00:00
require_once " autoload.php " ;
2010-11-10 10:14:44 +00:00
require_once " functions.php " ;
2007-03-01 08:54:55 +00:00
require_once " config.php " ;
2013-03-15 11:56:40 +00:00
require_once " sanity_check.php " ;
2007-03-01 08:54:55 +00:00
require_once " db.php " ;
2010-11-09 13:33:08 +00:00
require_once " db-prefs.php " ;
2012-10-23 20:25:56 +00:00
if ( ! defined ( 'PHP_EXECUTABLE' ))
define ( 'PHP_EXECUTABLE' , '/usr/bin/php' );
2017-12-02 08:42:42 +00:00
$pdo = Db :: pdo ();
2013-04-17 12:23:15 +00:00
init_plugins ();
2012-12-24 05:52:15 +00:00
2013-03-21 10:48:47 +00:00
$longopts = array ( " feeds " ,
" feedbrowser " ,
" daemon " ,
" daemon-loop " ,
" task: " ,
" cleanup-tags " ,
" quiet " ,
2013-03-21 10:56:04 +00:00
" log: " ,
2013-03-21 10:48:47 +00:00
" indexes " ,
2013-04-17 14:19:26 +00:00
" pidlock: " ,
2013-04-04 15:15:14 +00:00
" update-schema " ,
2013-03-21 10:48:47 +00:00
" convert-filters " ,
" force-update " ,
2015-08-04 10:32:52 +00:00
" gen-search-idx " ,
2013-03-21 10:48:47 +00:00
" list-plugins " ,
2016-01-17 11:24:20 +00:00
" debug-feed: " ,
" force-refetch " ,
" force-rehash " ,
2017-01-07 11:25:46 +00:00
" decrypt-feeds " ,
2013-03-21 10:48:47 +00:00
" help " );
2013-04-18 08:27:34 +00:00
foreach ( PluginHost :: getInstance () -> get_commands () as $command => $data ) {
2013-03-28 14:37:36 +00:00
array_push ( $longopts , $command . $data [ " suffix " ]);
2013-03-21 10:48:47 +00:00
}
$options = getopt ( " " , $longopts );
2012-10-23 20:25:56 +00:00
2014-02-06 19:03:05 +00:00
if ( ! is_array ( $options )) {
die ( " error: getopt() failed. " .
" Most probably you are using PHP CGI to run this script " .
" instead of required PHP CLI. Check tt-rss wiki page on updating feeds for " .
" additional information. \n " );
}
2013-03-21 10:48:47 +00:00
if ( count ( $options ) == 0 && ! defined ( 'STDIN' )) {
2012-08-23 09:17:22 +00:00
?> <html>
< head >
2012-10-23 20:25:56 +00:00
< title > Tiny Tiny RSS data update script .</ title >
2012-08-23 09:17:22 +00:00
< meta http - equiv = " Content-Type " content = " text/html; charset=utf-8 " >
2013-05-19 17:22:01 +00:00
< link rel = " stylesheet " type = " text/css " href = " css/utility.css " >
2012-08-23 09:17:22 +00:00
</ head >
< body >
2013-03-27 06:59:26 +00:00
< div class = " floatingLogo " >< img src = " images/logo_small.png " ></ div >
2012-10-23 20:25:56 +00:00
< h1 >< ? php echo __ ( " Tiny Tiny RSS data update script. " ) ?> </h1>
2012-08-23 09:17:22 +00:00
2017-02-23 10:06:39 +00:00
< ? php print_error ( " Please run this script from the command line. Use option \" --help \" to display command help if this error is displayed erroneously. " ); ?>
2012-08-23 09:17:22 +00:00
</ body ></ html >
< ? php
exit ;
}
2013-03-21 10:48:47 +00:00
if ( count ( $options ) == 0 || isset ( $options [ " help " ]) ) {
2010-11-09 13:33:08 +00:00
print " Tiny Tiny RSS data update script. \n \n " ;
print " Options: \n " ;
2013-03-21 10:48:47 +00:00
print " --feeds - update feeds \n " ;
print " --feedbrowser - update feedbrowser \n " ;
print " --daemon - start single-process update daemon \n " ;
print " --task N - create lockfile using this task id \n " ;
print " --cleanup-tags - perform tags table maintenance \n " ;
2013-03-21 11:05:57 +00:00
print " --quiet - don't output messages to stdout \n " ;
2013-03-21 10:56:04 +00:00
print " --log FILE - log messages to FILE \n " ;
2013-03-21 10:48:47 +00:00
print " --indexes - recreate missing schema indexes \n " ;
2013-04-04 15:15:14 +00:00
print " --update-schema - update database schema \n " ;
2015-08-04 10:32:52 +00:00
print " --gen-search-idx - generate basic PostgreSQL fulltext search index \n " ;
2013-03-21 10:48:47 +00:00
print " --convert-filters - convert type1 filters to type2 \n " ;
print " --force-update - force update of all feeds \n " ;
print " --list-plugins - list all available plugins \n " ;
2016-01-17 11:24:20 +00:00
print " --debug-feed N - perform debug update of feed N \n " ;
print " --force-refetch - debug update: force refetch feed data \n " ;
print " --force-rehash - debug update: force rehash articles \n " ;
2017-01-07 11:25:46 +00:00
print " --decrypt-feeds - decrypt feed passwords \n " ;
2013-03-21 10:48:47 +00:00
print " --help - show this help \n " ;
2012-12-24 05:52:15 +00:00
print " Plugin options: \n " ;
2013-04-18 08:27:34 +00:00
foreach ( PluginHost :: getInstance () -> get_commands () as $command => $data ) {
2013-03-28 14:37:36 +00:00
$args = $data [ 'arghelp' ];
printf ( " --%-19s - %s \n " , " $command $args " , $data [ " description " ]);
2012-12-24 05:52:15 +00:00
}
2010-11-09 13:33:08 +00:00
return ;
2007-03-01 08:54:55 +00:00
}
2007-05-04 03:15:58 +00:00
2013-04-17 08:36:27 +00:00
if ( ! isset ( $options [ 'daemon' ])) {
require_once " errorhandler.php " ;
}
2013-04-17 08:10:35 +00:00
if ( ! isset ( $options [ 'update-schema' ])) {
2013-04-17 12:23:15 +00:00
$schema_version = get_schema_version ();
2013-04-17 08:10:35 +00:00
if ( $schema_version != SCHEMA_VERSION ) {
die ( " Schema version is wrong, please upgrade the database. \n " );
}
}
2013-03-21 11:05:57 +00:00
define ( 'QUIET' , isset ( $options [ 'quiet' ]));
2013-03-21 10:56:04 +00:00
if ( isset ( $options [ " log " ])) {
_debug ( " Logging to " . $options [ " log " ]);
define ( 'LOGFILE' , $options [ " log " ]);
}
2013-03-21 10:48:47 +00:00
if ( ! isset ( $options [ " daemon " ])) {
2010-11-09 13:33:08 +00:00
$lock_filename = " update.lock " ;
} else {
$lock_filename = " update_daemon.lock " ;
}
2007-02-24 16:46:39 +00:00
2013-03-21 10:48:47 +00:00
if ( isset ( $options [ " task " ])) {
_debug ( " Using task id " . $options [ " task " ]);
$lock_filename = $lock_filename . " -task_ " . $options [ " task " ];
}
2013-04-17 14:19:26 +00:00
if ( isset ( $options [ " pidlock " ])) {
$my_pid = $options [ " pidlock " ];
$lock_filename = " update_daemon- $my_pid .lock " ;
}
_debug ( " Lock: $lock_filename " );
2010-11-09 13:33:08 +00:00
$lock_handle = make_lockfile ( $lock_filename );
$must_exit = false ;
2007-02-24 16:46:39 +00:00
2013-04-17 18:09:11 +00:00
if ( isset ( $options [ " task " ]) && isset ( $options [ " pidlock " ])) {
2013-04-17 14:19:26 +00:00
$waits = $options [ " task " ] * 5 ;
_debug ( " Waiting before update ( $waits ) " );
sleep ( $waits );
}
2010-11-09 13:33:08 +00:00
// Try to lock a file in order to avoid concurrent update.
if ( ! $lock_handle ) {
die ( " error: Can't create lockfile ( $lock_filename ). " .
" Maybe another update process is already running. \n " );
}
2007-02-24 16:46:39 +00:00
2013-03-21 10:48:47 +00:00
if ( isset ( $options [ " force-update " ])) {
_debug ( " marking all feeds as needing update... " );
2017-12-02 08:42:42 +00:00
$pdo -> query ( " UPDATE ttrss_feeds SET
last_update_started = '1970-01-01' , last_updated = '1970-01-01' " );
2013-03-21 10:48:47 +00:00
}
if ( isset ( $options [ " feeds " ])) {
2017-05-05 15:10:07 +00:00
RSSUtils :: update_daemon_common ();
RSSUtils :: housekeeping_common ( true );
2011-04-21 04:51:59 +00:00
2013-04-18 08:27:34 +00:00
PluginHost :: getInstance () -> run_hooks ( PluginHost :: HOOK_UPDATE_TASK , " hook_update_task " , $op );
2010-11-09 13:33:08 +00:00
}
2007-02-24 16:46:39 +00:00
2013-03-21 10:48:47 +00:00
if ( isset ( $options [ " feedbrowser " ])) {
2017-05-05 15:10:07 +00:00
$count = RSSUtils :: update_feedbrowser_cache ();
2010-11-09 13:33:08 +00:00
print " Finished, $count feeds processed. \n " ;
2007-02-24 16:46:39 +00:00
}
2010-11-09 13:33:08 +00:00
2013-03-21 10:48:47 +00:00
if ( isset ( $options [ " daemon " ])) {
2010-11-09 13:33:08 +00:00
while ( true ) {
2013-04-01 15:51:43 +00:00
$quiet = ( isset ( $options [ " quiet " ])) ? " --quiet " : " " ;
2013-09-02 06:54:20 +00:00
$log = isset ( $options [ 'log' ]) ? '--log ' . $options [ 'log' ] : '' ;
2013-04-01 15:51:43 +00:00
2013-09-02 06:54:20 +00:00
passthru ( PHP_EXECUTABLE . " " . $argv [ 0 ] . " --daemon-loop $quiet $log " );
2017-05-06 07:54:14 +00:00
// let's enforce a minimum spawn interval as to not forkbomb the host
$spawn_interval = max ( 60 , DAEMON_SLEEP_INTERVAL );
_debug ( " Sleeping for $spawn_interval seconds... " );
sleep ( $spawn_interval );
2007-02-24 16:46:39 +00:00
}
2007-03-01 08:54:55 +00:00
}
2007-05-05 11:07:38 +00:00
2013-03-21 10:48:47 +00:00
if ( isset ( $options [ " daemon-loop " ])) {
2010-11-09 13:33:08 +00:00
if ( ! make_stampfile ( 'update_daemon.stamp' )) {
2013-03-30 12:28:21 +00:00
_debug ( " warning: unable to create stampfile \n " );
2010-11-09 13:33:08 +00:00
}
2007-05-05 11:07:38 +00:00
2017-05-05 15:10:07 +00:00
RSSUtils :: update_daemon_common ( isset ( $options [ " pidlock " ]) ? 50 : DAEMON_FEED_LIMIT );
2013-05-20 10:39:14 +00:00
if ( ! isset ( $options [ " pidlock " ]) || $options [ " task " ] == 0 )
2017-05-05 15:10:07 +00:00
RSSUtils :: housekeeping_common ( true );
2011-04-12 15:33:12 +00:00
2013-04-24 09:43:40 +00:00
PluginHost :: getInstance () -> run_hooks ( PluginHost :: HOOK_UPDATE_TASK , " hook_update_task " , $op );
2007-02-24 16:46:39 +00:00
}
2013-03-21 10:48:47 +00:00
if ( isset ( $options [ " cleanup-tags " ])) {
2013-04-17 12:23:15 +00:00
$rc = cleanup_tags ( 14 , 50000 );
2012-08-04 18:15:40 +00:00
_debug ( " $rc tags deleted. \n " );
2010-11-13 16:59:42 +00:00
}
2013-03-21 10:48:47 +00:00
if ( isset ( $options [ " indexes " ])) {
2012-08-09 10:20:27 +00:00
_debug ( " PLEASE BACKUP YOUR DATABASE BEFORE PROCEEDING! " );
_debug ( " Type 'yes' to continue. " );
if ( read_stdin () != 'yes' )
exit ;
_debug ( " clearing existing indexes... " );
if ( DB_TYPE == " pgsql " ) {
2017-12-02 08:42:42 +00:00
$sth = $pdo -> query ( " SELECT relname FROM
2012-08-09 10:20:27 +00:00
pg_catalog . pg_class WHERE relname LIKE 'ttrss_%'
AND relname NOT LIKE '%_pkey'
AND relkind = 'i' " );
} else {
2017-12-02 08:42:42 +00:00
$sth = $pdo -> query ( " SELECT index_name,table_name FROM
2012-08-09 10:20:27 +00:00
information_schema . statistics WHERE index_name LIKE 'ttrss_%' " );
}
2017-12-02 08:42:42 +00:00
while ( $line = $sth -> fetch ()) {
2012-08-09 10:20:27 +00:00
if ( DB_TYPE == " pgsql " ) {
$statement = " DROP INDEX " . $line [ " relname " ];
_debug ( $statement );
} else {
$statement = " ALTER TABLE " .
$line [ 'table_name' ] . " DROP INDEX " . $line [ 'index_name' ];
_debug ( $statement );
}
2017-12-02 08:42:42 +00:00
$pdo -> query ( $statement );
2012-08-09 10:20:27 +00:00
}
_debug ( " reading indexes from schema for: " . DB_TYPE );
$fp = fopen ( " schema/ttrss_schema_ " . DB_TYPE . " .sql " , " r " );
if ( $fp ) {
while ( $line = fgets ( $fp )) {
$matches = array ();
if ( preg_match ( " /^create index ([^ ]+) on ([^ ]+) $ /i " , $line , $matches )) {
$index = $matches [ 1 ];
$table = $matches [ 2 ];
$statement = " CREATE INDEX $index ON $table " ;
_debug ( $statement );
2017-12-02 08:42:42 +00:00
$pdo -> query ( $statement );
2012-08-09 10:20:27 +00:00
}
}
fclose ( $fp );
} else {
_debug ( " unable to open schema file. " );
}
_debug ( " all done. " );
}
2013-03-21 10:48:47 +00:00
if ( isset ( $options [ " convert-filters " ])) {
2012-08-30 14:50:56 +00:00
_debug ( " WARNING: this will remove all existing type2 filters. " );
_debug ( " Type 'yes' to continue. " );
if ( read_stdin () != 'yes' )
exit ;
_debug ( " converting filters... " );
2017-12-02 08:42:42 +00:00
$pdo -> query ( " DELETE FROM ttrss_filters2 " );
2012-08-30 14:50:56 +00:00
2017-12-02 08:42:42 +00:00
$res = $pdo -> query ( " SELECT * FROM ttrss_filters ORDER BY id " );
2012-08-30 14:50:56 +00:00
2017-12-02 08:42:42 +00:00
while ( $line = $res -> fetch ()) {
2012-08-30 14:50:56 +00:00
$owner_uid = $line [ " owner_uid " ];
2012-08-31 08:54:37 +00:00
// date filters are removed
if ( $line [ " filter_type " ] != 5 ) {
$filter = array ();
if ( sql_bool_to_bool ( $line [ " cat_filter " ])) {
$feed_id = " CAT: " . ( int ) $line [ " cat_id " ];
} else {
$feed_id = ( int ) $line [ " feed_id " ];
}
2012-08-30 14:50:56 +00:00
2012-08-31 08:54:37 +00:00
$filter [ " enabled " ] = $line [ " enabled " ] ? " on " : " off " ;
$filter [ " rule " ] = array (
json_encode ( array (
" reg_exp " => $line [ " reg_exp " ],
" feed_id " => $feed_id ,
" filter_type " => $line [ " filter_type " ])));
2012-08-30 14:50:56 +00:00
2012-08-31 08:54:37 +00:00
$filter [ " action " ] = array (
json_encode ( array (
" action_id " => $line [ " action_id " ],
" action_param_label " => $line [ " action_param " ],
" action_param " => $line [ " action_param " ])));
2012-08-30 14:50:56 +00:00
2012-08-31 08:54:37 +00:00
// Oh god it's full of hacks
2012-08-31 08:24:13 +00:00
2012-08-31 08:54:37 +00:00
$_REQUEST = $filter ;
$_SESSION [ " uid " ] = $owner_uid ;
2012-08-30 14:50:56 +00:00
2013-04-18 19:19:14 +00:00
$filters = new Pref_Filters ( $_REQUEST );
2012-08-31 08:54:37 +00:00
$filters -> add ();
}
2012-08-30 14:50:56 +00:00
}
}
2013-04-04 15:15:14 +00:00
if ( isset ( $options [ " update-schema " ])) {
_debug ( " checking for updates ( " . DB_TYPE . " )... " );
2017-12-01 22:28:30 +00:00
$updater = new DbUpdater ( Db :: pdo (), DB_TYPE , SCHEMA_VERSION );
2013-04-04 15:15:14 +00:00
if ( $updater -> isUpdateRequired ()) {
_debug ( " schema update required, version " . $updater -> getSchemaVersion () . " to " . SCHEMA_VERSION );
_debug ( " WARNING: please backup your database before continuing. " );
_debug ( " Type 'yes' to continue. " );
if ( read_stdin () != 'yes' )
exit ;
for ( $i = $updater -> getSchemaVersion () + 1 ; $i <= SCHEMA_VERSION ; $i ++ ) {
_debug ( " performing update up to version $i ... " );
2016-04-26 17:04:24 +00:00
$result = $updater -> performUpdateTo ( $i , false );
2013-04-04 15:15:14 +00:00
_debug ( $result ? " OK! " : " FAILED! " );
if ( ! $result ) return ;
}
} else {
_debug ( " update not required. " );
}
}
2015-08-04 10:32:52 +00:00
if ( isset ( $options [ " gen-search-idx " ])) {
echo " Generating search index (stemming set to English)... \n " ;
2015-08-04 09:52:49 +00:00
2017-12-02 08:42:42 +00:00
$res = $pdo -> query ( " SELECT COUNT(id) AS count FROM ttrss_entries WHERE tsvector_combined IS NULL " );
$row = $res -> fetch ();
$count = $row [ 'count' ];
2015-08-04 09:52:49 +00:00
2015-08-04 11:36:39 +00:00
print " Articles to process: $count . \n " ;
2015-08-04 09:52:49 +00:00
2015-08-04 11:36:39 +00:00
$limit = 500 ;
$processed = 0 ;
2015-08-04 09:52:49 +00:00
2017-12-02 08:42:42 +00:00
$sth = $pdo -> prepare ( " SELECT id, title, content FROM ttrss_entries WHERE
tsvector_combined IS NULL ORDER BY id LIMIT ? " );
$sth -> execute ([ $limit ]);
$usth = $pdo -> prepare ( " UPDATE ttrss_entries
SET tsvector_combined = to_tsvector ( 'english' , ? ) WHERE id = ? " );
2015-08-04 09:52:49 +00:00
while ( true ) {
2017-12-02 08:42:42 +00:00
while ( $line = $sth -> fetch ()) {
2017-12-03 07:10:01 +00:00
$tsvector_combined = mb_substr ( $line [ 'title' ] . ' ' .
preg_replace ( '/[<\?\:]/' , ' ' , strip_tags ( $line [ 'content' ])),
2017-12-02 08:42:42 +00:00
0 , 1000000 );
2015-08-04 09:52:49 +00:00
2017-12-02 08:42:42 +00:00
$usth -> execute ([ $tsvector_combined , $line [ 'id' ]]);
$processed ++ ;
2015-08-05 13:58:07 +00:00
}
2015-08-04 09:52:49 +00:00
2015-08-05 13:58:07 +00:00
print " Processed $processed articles... \n " ;
2015-08-04 11:36:39 +00:00
2017-12-02 08:42:42 +00:00
if ( $processed < $limit ) {
2015-08-04 09:52:49 +00:00
echo " All done. \n " ;
break ;
}
}
}
2013-03-21 10:48:47 +00:00
if ( isset ( $options [ " list-plugins " ])) {
2013-05-07 07:35:10 +00:00
$tmppluginhost = new PluginHost ();
2015-10-08 14:02:32 +00:00
$tmppluginhost -> load_all ( $tmppluginhost :: KIND_ALL , false );
2012-12-25 14:57:02 +00:00
$enabled = array_map ( " trim " , explode ( " , " , PLUGINS ));
echo " List of all available plugins: \n " ;
2012-12-24 11:39:42 +00:00
foreach ( $tmppluginhost -> get_plugins () as $name => $plugin ) {
2012-12-25 06:02:08 +00:00
$about = $plugin -> about ();
2012-12-24 11:39:42 +00:00
2012-12-25 14:57:02 +00:00
$status = $about [ 3 ] ? " system " : " user " ;
if ( in_array ( $name , $enabled )) $name .= " * " ;
printf ( " %-50s %-10s v%.2f (by %s) \n %s \n \n " ,
$name , $status , $about [ 0 ], $about [ 2 ], $about [ 1 ]);
2012-12-24 11:39:42 +00:00
}
2012-12-25 14:57:02 +00:00
echo " Plugins marked by * are currently enabled for all users. \n " ;
2012-12-24 11:39:42 +00:00
}
2016-01-17 11:24:20 +00:00
if ( isset ( $options [ " debug-feed " ])) {
$feed = $options [ " debug-feed " ];
if ( isset ( $options [ " force-refetch " ])) $_REQUEST [ " force_refetch " ] = true ;
if ( isset ( $options [ " force-rehash " ])) $_REQUEST [ " force_rehash " ] = true ;
$_REQUEST [ 'xdebug' ] = 1 ;
2017-05-05 15:10:07 +00:00
$rc = RSSUtils :: update_rss_feed ( $feed ) != false ? 0 : 1 ;
2017-04-25 10:39:01 +00:00
exit ( $rc );
2016-01-17 11:24:20 +00:00
}
2017-01-07 11:25:46 +00:00
if ( isset ( $options [ " decrypt-feeds " ])) {
if ( ! function_exists ( " mcrypt_decrypt " )) {
_debug ( " mcrypt functions not available. " );
return ;
}
2017-12-02 08:42:42 +00:00
$res = $pdo -> query ( " SELECT id, auth_pass FROM ttrss_feeds WHERE auth_pass_encrypted = true " );
2017-01-07 11:25:46 +00:00
require_once " crypt.php " ;
$total = 0 ;
2017-12-02 08:42:42 +00:00
$pdo -> beginTransaction ();
$usth = $pdo -> prepare ( " UPDATE ttrss_feeds SET auth_pass_encrypted = false, auth_pass = ?
WHERE id = ? " );
2017-01-07 11:25:46 +00:00
2017-12-02 08:42:42 +00:00
while ( $line = $res -> fetch ()) {
2017-01-07 11:25:46 +00:00
_debug ( " processing feed id " . $line [ " id " ]);
2017-12-02 08:42:42 +00:00
$auth_pass = decrypt_string ( $line [ " auth_pass " ]);
2017-01-07 11:25:46 +00:00
2017-12-02 08:42:42 +00:00
$usth -> execute ([ $auth_pass , $line [ 'id' ]]);
2017-01-07 11:25:46 +00:00
++ $total ;
}
2017-12-02 08:42:42 +00:00
$pdo -> commit ();
2017-01-07 11:25:46 +00:00
_debug ( " $total feeds processed. " );
}
2013-04-18 08:27:34 +00:00
PluginHost :: getInstance () -> run_commands ( $options );
2012-12-24 05:52:15 +00:00
2017-01-06 17:06:09 +00:00
if ( file_exists ( LOCK_DIRECTORY . " / $lock_filename " ))
if ( strtoupper ( substr ( PHP_OS , 0 , 3 )) == 'WIN' )
fclose ( $lock_handle );
2013-06-07 05:27:52 +00:00
unlink ( LOCK_DIRECTORY . " / $lock_filename " );
2013-04-12 05:55:41 +00:00
?>
