diff --git a/config-example.php b/config-example.php index ad6c7c5..a50b62f 100644 --- a/config-example.php +++ b/config-example.php @@ -1,7 +1,12 @@ set( 'wiki', 'itwiki' ) + ->set( 'seealso', 'Voci correlate' ); diff --git a/includes/autoload.php b/includes/autoload.php index dd2ed5e..fd744e6 100644 --- a/includes/autoload.php +++ b/includes/autoload.php @@ -1,20 +1,24 @@ . // load some common functions require __DIR__ . '/functions.php'; // autoload boz-mw classes require __DIR__ . '/boz-mw/autoload.php'; + +// load the dummy config class +require __DIR__ . '/class-Config.php'; + diff --git a/includes/class-Config.php b/includes/class-Config.php new file mode 100644 index 0000000..8cd987a --- /dev/null +++ b/includes/class-Config.php @@ -0,0 +1,69 @@ +. + +namespace orphanizerbot; + +/** + * Dummy class used to store static config + */ +class Config { + + /** + * Get the singleton instance + * + * @return self + */ + public static function instance() { + static $self = false; + if( ! $self ) { + $self = new self(); + } + return $self; + } + + /** + * Set a configuration value + * + * @param $name string + * @param $v mixed + * @return self + */ + public function set( $name, $v ) { + $this->$name = $v; + return $this; + } + + /** + * Check existence of a configuration + * + * @param $name string + * @return bool + */ + public function has( $name ) { + return isset( $this->$name ); + } + + /** + * Get a configuration value or a default one + * + * @param $name string + * @param $default mixed + * @return mixed + */ + public function get( $name, $default = null ) { + return $this->has( $name ) ? $this->$name : $default; + } + +} diff --git a/includes/functions.php b/includes/functions.php index 61a92b8..9ebf928 100644 --- a/includes/functions.php +++ b/includes/functions.php @@ -1,146 +1,156 @@ . use \cli\Log; use \cli\Opts; +use \orphanizerbot\Config; /** * Show the help message and die */ function show_help() { global $argv; echo "Welcome in your MediaWiki Orphanizer bot!\n\n"; echo "Available options, most of them also on-wiki:\n"; // show every available option foreach( Opts::instance()->getParams() as $param ) { // long option if( $param->hasLongName() ) { echo " --{$param->getLongName()}"; if( $param->isValueRequired() ) { echo "=VALUE"; } echo "\n"; } // short option if( $param->hasShortName() ) { echo " -{$param->getShortName()}"; if( $param->isValueRequired() ) { echo "=VALUE"; } echo "\n"; } // description echo " \t{$param->getDescription()}\n"; } echo "\n Example:\n" . " \t{$argv[0]} --wiki=itwiki --list=Wikipedia:PDC/Elenco\n\n" . " Have fun! by Valerio Bozzolan, Daimona Eaytoy\n" ; exit( 0 ); } /** * Get an option value * - * It's retrieved from a command line argument, - * or from the on-wiki configuration page, - * or from the default value provided. + * It's retrieved with this priority from greater to lower: + * 1. the command line argument + * 2. the on-wiki configuration page + * 3. from the static configuration + * 4. from the default value provided * * @param $name string * @param $default mixed Default value */ function option( $name, $default = null ) { - // try the wiki option, then the $default - $value = wiki_option( $name, $default ); + $value = $default; - // try the cli option, then wiki option - return Opts::instance()->getArg( $name, $value ); + // try the hardcooded option first, then the $default + $value = Config::instance()->get( $name, $value ); + + // try the wiki option, then the hardcoded option + $value = wiki_option( $name, $value ); + + // try the cli option, then the wiki option + $value = Opts::instance()->getArg( $name, $value ); + + return $value; } /** * Get an on-wiki option value * * The option is retrieved from a wiki page * with a JSON content model. * * @param $name string * @param $default mixed Default value */ function wiki_option( $name, $default = null ) { $config = wiki_config(); // retrieve the option if exists return isset( $config->{ $name } ) ? $config->{ $name } : $default; } /** * Load the wiki configuration (once) * * @return object */ function wiki_config() { static $config = null; // retrieve the configuration only once if( $config === null ) { $page = Opts::instance()->getArg( 'cfg', 'Utente:OrfanizzaBot/Configurazione' ); $config = fetch_json_page( $page ); } return $config; } /** * Fetch a MediaWiki page with the JSON * content model. * * @param $pagename string * @return object */ function fetch_json_page( $pagename ) { global $wiki; Log::info( "reading $pagename" ); $response = $wiki->fetch( [ 'action' => 'query', 'titles' => $pagename, 'prop' => 'revisions', 'rvslots' => 'main', 'rvprop' => 'content', ] ); $revision = reset( $response->query->pages )->revisions[0]; if ( $revision->slots->main->contentmodel !== 'json' ) { Log::error( "the page $pagename must have JSON content model" ); exit( 1 ); } return json_decode( $revision->slots->main->{ '*' } ); } diff --git a/orphanize.php b/orphanize.php index c1d3d01..9317f08 100755 --- a/orphanize.php +++ b/orphanize.php @@ -1,424 +1,433 @@ #!/usr/bin/php . -namespace itwikidelbot; +namespace orphanizerbot; // die on whatever error set_error_handler( function( $errno, $errstr, $errfile, $errline ) { if( error_reporting() !== 0 ) { throw new \ErrorException( $errstr, 0, $errno, $errfile, $errline ); } } ); // do not expose from web isset( $argv[ 0 ] ) or exit( 1 ); // autoload classes require __DIR__ . '/includes/autoload.php'; // require config file file_exists( $config_path = __DIR__ . '/config.php' ) or die( "Please provide the configuration file at $config_path\n" ); require $config_path; // how much titles at time requesting - this is a MediaWiki limit define( 'MAX_TRANCHE_TITLES', 50 ); // classes used use \cli\Input; use \cli\Opts; use \cli\ParamFlag; use \cli\ParamValued; use \cli\Log; use \web\MediaWikis; use \mw\Wikilink; use \mw\Ns; use \mw\API\ProtectedPageException; use \regex\Generic as Regex; // register available options $opts = Opts::instance()->register( [ // register arguments with a value new ParamValued( 'wiki', null, 'Specify a wiki from its UID' ), new ParamValued( 'cfg', null, 'Title of an on-wiki configuration page with JSON content model' ), new ParamValued( 'list', null, 'Specify a pagename that should contain the wikilinks to be orphanized' ), new ParamValued( 'summary', null, 'Edit summary' ), new ParamValued( 'list-summary', null, 'Edit summary for editing the list' ), new ParamValued( 'done-text', null, 'Replacement for the wikilink in the list' ), new ParamValued( 'ns', null, 'Namespace whitelist' ), new ParamValued( 'delay', null, 'Additional delay between each edit' ), new ParamValued( 'warmup', null, 'Start only if the last edit on the list was done at least $warmup seconds ago' ), new ParamValued( 'cooldown', null, 'End early when reaching this number of edits' ), new ParamValued( 'seealso', null, 'Title of your local "See also" section' ), // register arguments without a value new ParamFlag( 'debug', null, 'Increase verbosity' ), new ParamFlag( 'help', 'h', 'Show this message and quit' ), new ParamFlag( 'no-interaction', null, 'Do not confirm every change' ), ] ); // show help screen if( $opts->getArg( 'help' ) ) { show_help(); } // cli-only parameters $NO_INTERACTION = $opts->getArg( 'no-interaction' ); $TITLE_SOURCE = $opts->getArg( 'list', 'Utente:OrfanizzaBot/Wikilink da orfanizzare' ); Log::info( "start" ); // increase verbosity if( $opts->getArg( 'debug' ) ) { Log::$DEBUG = true; } + +// wiki uid (from command line or from configuration file) +$wiki_uid = Config::instance()->get( 'wiki' ); +$wiki_uid = $opts->getArg( 'wiki', $wiki_uid ); +if( ! $wiki_uid ) { + Log::error( "please choose the wiki! exit" ); + exit( 1 ); +} + // wiki instance -$wiki = Mediawikis::findFromUid( $opts->getArg( 'wiki', 'itwiki' ) ); +$wiki = Mediawikis::findFromUid( $wiki_uid ); // load the wiki config wiki_config(); // parameters available both from cli and on-wiki $SUMMARY = option( 'summary', "Bot: pages orphanization" ); $LIST_SUMMARY = option( 'list-summary', "Bot: orphanization list update" ); $DONE_TEXT = option( 'done-text', "* [[Special:WhatLinksHere/$1]] - {{done}}" ); $NS = option( 'ns' ); $WARMUP = option( 'warmup', -1 ); $COOLDOWN = option( 'cooldown', 1000 ); $DELAY = option( 'delay', 0 ); $SEEALSO = option( 'seealso', 'See also' ); // query titles to be orphanized alongside the last revision of the list $responses = $wiki->createQuery( [ 'action' => 'query', 'titles' => $TITLE_SOURCE, 'prop' => [ 'links', 'revisions', ], 'rvslots' => 'main', 'rvprop' => 'timestamp', ] ); // collect links and take the last edit timestamp $titles_to_be_orphanized = []; Log::info( "reading $TITLE_SOURCE" ); foreach( $responses as $response ) { foreach( $response->query->pages as $page ) { // check if list is unexisting if( isset( $page->missing ) ) { Log::error( "missing list $TITLE_SOURCE" ); exit( 1 ); } // check warmup $timestamp = reset( $page->revisions )->timestamp; $timestamp = \DateTime::createFromFormat( \DateTime::ISO8601, $timestamp ); $seconds = time() - $timestamp->getTimestamp(); if( $seconds < $WARMUP ) { Log::info( "edited just $seconds seconds ago: quit until warmup $WARMUP" ); exit( 1 ); } // collect links (if any) if( isset( $page->links ) ) { foreach( $page->links as $link ) { $titles_to_be_orphanized[] = $link->title; } } } } // die if no links if( ! $titles_to_be_orphanized ) { Log::info( 'empty list' ); exit( 1 ); } // keep a copy $involved_pagetitles = $titles_to_be_orphanized; // log titles Log::info( 'read ' . count( $titles_to_be_orphanized ) . ' pages to be orphanized:' ); foreach( $titles_to_be_orphanized as $title ) { Log::info( " $title" ); } // associative array of page IDs as key and a boolean as value containg pages to be orphanized $involved_pageids = []; // note that the API accepts a maximum trance of titles while( $less_titles_to_be_orphanized = array_splice( $titles_to_be_orphanized, 0, MAX_TRANCHE_TITLES ) ) { // API arguments for the linkshere query $linksto_args = [ 'action' => 'query', 'titles' => $less_titles_to_be_orphanized, 'prop' => 'linkshere', 'lhprop' => [ 'pageid', 'title', ], 'lhshow' => '!redirect', 'lhlimit' => 300, ]; // limit to certain namespaces if( $NS !== null ) { $linksto_args[ 'lhnamespace' ] = implode( '|', $NS ); } // cumulate the linkshere page ids Log::info( "requesting linkshere..." ); $linksto = $wiki->createQuery( $linksto_args ); foreach( $linksto as $response ) { foreach( $response->query->pages as $page ) { if( isset( $page->linkshere ) ) { foreach( $page->linkshere as $linkingpage ) { if( $linkingpage->title !== $TITLE_SOURCE ) { $involved_pageids[] = (int) $linkingpage->pageid; } } } } } } // count of involved pages Log::info( sprintf( "found %d pages containing the %d involved wlinks", count( $involved_pageids ), count( $involved_pagetitles ) ) ); // number of edited pages $edits = 0; // note that the API accepts a maximum tranche of IDs while( $less_involved_pageids = array_splice( $involved_pageids, 0, MAX_TRANCHE_TITLES ) ) { // query last revision $responses = $wiki->createQuery( [ 'action' => 'query', 'pageids' => $less_involved_pageids, 'prop' => 'revisions', 'rvslots' => 'main', 'rvprop' => [ 'content', 'timestamp', ], ] ); // for each response foreach( $responses as $response ) { // for each page foreach( $response->query->pages as $page ) { // avoid too many edits if( $edits > $COOLDOWN ) { Log::info( "reached cooldown: stop" ); exit( 0 ); } // page ID to be edited $pageid = $page->pageid; // does it have a revision? if( !isset( $page->revisions[ 0 ] ) ) { continue; } // the first revision $revision = $page->revisions[ 0 ]; // timestamp of the revision useful to avoid edit conflicts $timestamp = $revision->timestamp; // wikitext from the main slot of this revision $wikitext_raw = $revision->slots->main->{ '*' }; // create a Wikitext object $wikitext = $wiki->createWikitext( $wikitext_raw ); // for each of the titles to be orphanized foreach( $involved_pagetitles as $involved_pagetitle ) { // parse the title being orphanized $title = $wiki->createTitleParsing( $involved_pagetitle ); // if it's a category, remove it if( $title->getNs()->getID() === 14 ) { $wikitext->removeCategory( $title->getTitle() ); } // a wikilink with and without alias $wikilink_simple = $wiki->createWikilink( $title, Wikilink::NO_ALIAS ); $wikilink_alias = $wiki->createWikilink( $title, Wikilink::WHATEVER_ALIAS ); // replace simple links e.g. [[Hello]] $wikilink_regex_simple = $wikilink_simple->getRegex( [ 'title-group-name' => 'title', ] ); // replace links with alias e.g. [[Hello|whatever]] $wikilink_regex_alias = $wikilink_alias->getRegex( [ 'alias-group-name' => 'alias', ] ); // replace entry from "See also" section $wikilink_regex_clean = $wikilink_simple->getRegex(); $wikilink_regex_clean = Regex::spaceBurger( $wikilink_regex_clean ); $seealso = preg_quote( $SEEALSO ); $seealso_regex = '/' . Regex::groupNamed( "\\n== *$seealso *== *((?!=).*\n)*", 'keep' ) . Regex::groupNamed( "[ \\t]*\*[ \\t]*{$wikilink_regex_clean}.*\\n", 'wlink' ) . '/'; Log::debug( "regex simple wikilink:" ); Log::debug( $wikilink_regex_simple ); Log::debug( "regex wikilink aliased:" ); Log::debug( $wikilink_regex_alias ); Log::debug( "regex see also:" ); Log::debug( $seealso_regex ); // strip out the entry from «See also» section $wikitext->pregReplaceCallback( $seealso_regex, function ( $matches ) { return $matches[ 'keep' ]; } ); // convert '[[Hello]]' to 'Hello' $wikitext->pregReplaceCallback( "/$wikilink_regex_simple/", function ( $matches ) { // fix unwanted indentations $title = ltrim( $matches[ 'title' ], ':' ); return trim( $title ); } ); // convert '[[Hello|world]]' to 'world' $wikitext->pregReplaceCallback( "/$wikilink_regex_alias/", function ( $matches ) { // fix unwanted indentations return trim( $matches[ 'alias' ] ); } ); } // end loop titles to be orphanized // check for changes and save if( $wikitext->isChanged() ) { Log::info( "changes on page $pageid:" ); foreach( $wikitext->getHumanUniqueSobstitutions() as $substitution ) { Log::info( "\t $substitution" ); } if( $NO_INTERACTION || 'n' !== Input::yesNoQuestion( "confirm changes" ) ) { try { // the entire world absolutely needs this shitty ASCII animation - trust me if( $edits && $DELAY ) { Log::info( "delay $DELAY seconds", [ 'newline' => false ] ); for( $i = 0; $i < $DELAY; $i++ ) { sleep( 1 ); echo '.'; } echo "\n"; } // eventually login and save $wiki->login()->edit( [ 'pageid' => $pageid, 'text' => $wikitext->getWikitext(), 'summary' => $SUMMARY, 'basetimestamp' => $timestamp, 'minor' => 1, 'bot' => 1, ] ); $edits++; } catch( ProtectedPageException $e ) { Log::warn( "skip protected page $pageid" ); } } // end confirmation } // end save } // end loop pages } // end loop responses } // end loop involved page IDs // done! remove the link from the list Log::info( "removing orphanized pages from list" ); // fetch a fresh version of the list $response = $wiki->fetch( [ 'action' => 'query', 'titles' => $TITLE_SOURCE, 'prop' => 'revisions', 'rvprop' => 'content', 'rvslots' => 'main', ] ); // content of the list $content = reset( $response->query->pages )->revisions[0]->slots->main->{ '*' }; $wikitext = $wiki->createWikitext( $content ); // remove each entry from the list foreach( $involved_pagetitles as $title_raw ) { $wlink = $wiki->createTitleParsing( $title_raw ) ->createWikilink( Wikilink::WHATEVER_ALIAS ) ->getRegex(); // strip out the whole related line and replace with something else $from = "/.*$wlink.*/"; // @todo In case done-text contains the full link to a page, and it has already been // replaced in a previous run, don't replace it again. $to = str_replace( '$1', $title_raw, $DONE_TEXT ); $wikitext->pregReplace( $from, $to ); } // update list if( $wikitext->isChanged() ) { try { $wiki->login()->edit( [ 'title' => $TITLE_SOURCE, 'text' => $wikitext->getWikitext(), 'summary' => $LIST_SUMMARY, 'bot' => 1, ] ); } catch( ProtectedPageException $e ) { Log::warn( "can't update list because of protection" ); } } Log::info( "end" );