diff --git a/README.md b/README.md index f97c6c0..569697b 100644 --- a/README.md +++ b/README.md @@ -1,63 +1,99 @@ # MediaWiki Orphanizer bot This software is an Italian Wikipedia bot. It delinks page titles. Actually this is just a proof-of-concept from an idea of [Parma1983](https://it.wikipedia.org/wiki/Utente:Parma1983) and [.avgas](https://it.wikipedia.org/wiki/Utente:.avgas) and other awesome wikiguys. ## Installation sudo apt install git php-cli git clone --recursive https://github.com/valerio-bozzolan/MediaWikiOrphanizerBot.git ## Configuration You know you should provide your bot credentials in order to use a tool. 1. Open the file [`config-example.php`](config-example.php) with a text editor 2. Fill your bot credentials 3. Save-as `config.php` ## On-wiki configuration You need two pages: one with a list of links pointing to the pages to orphanize (pass its title via the `list` parameter), and one with generic config (to be passed via `cfg`). The latter should be a JSON page and can have the following options: - * `summary` (string) - The summary to use when editing. - * `ns` (array) - To only edit pages on the given namespaces. `null` means all namespaces. +* `summary` (string) - The summary to use when editing. +* `ns` (array) - To only edit pages on the given namespaces. `null` means all namespaces. +* `warmup` (int) - number of __seconds__ to wait before starting (after last edit on the list) +* `cooldown` (int) - number of __edits__ to do until shutdown (you may want to re-schedule) +* `delay` (int) - number of __seconds__ to wait before saving -## Usage +An example: + +``` +{ + "summary": "Bot: orphanizing links", + "ns": [ + 0 + ], + "warmup": 120, + "cooldown": 10, + "delay": 30 +} +``` + +## Usage from command line ./orphanizer.php --help ``` Welcome in your MediaWiki Orphanizer bot! - Usage: ./orphanize.php [OPTIONS] - Options: --wiki UID Specify a wiki from it's UID. - --list PAGENAME Specify a pagename that should - contain the wikilinks to be - orphanized by this bot. - --cfg PAGENAME Read the config from the specified - wikipage - --help Show this message and quit. +Available options, most of them also on-wiki: + --wiki=VALUE + Specify a wiki from it's UID + --cfg=VALUE + Title of an on-wiki configuration page with JSON content model + --list=VALUE + Specify a pagename that should contain the wikilinks to be orphanized + --summary=VALUE + Edit summary + --ns=VALUE + Namespace whitelist + --delay=VALUE + Additional delay between each edit + --warmup=VALUE + Start only if the last edit on the list was done at least $warmup seconds ago + --cooldown=VALUE + End early when reaching this number of edits + --seealso=VALUE + Title of your local "See also" section + --debug + Increase verbosity + --help + -h + Show this message and quit + --no-interaction + Do not confirm every change + Example: - ./orphanize.php --wiki itwiki --list Wikipedia:PDC/Elenco + ./orphanize.php --wiki=itwiki --list=Wikipedia:PDC/Elenco - Have fun! by Valerio Bozzolan + Have fun! by Valerio Bozzolan, Daimona Eaytoy ``` ## License Copyright (C) 2019 [Valerio Bozzolan](https://it.wikipedia.org/wiki/Utente:Valerio_Bozzolan) This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more details. You should have received a copy of the GNU Affero General Public License along with this program. If not, see . diff --git a/config-example.php b/config-example.php index 76020a8..ad6c7c5 100644 --- a/config-example.php +++ b/config-example.php @@ -1,5 +1,7 @@ . +// load some common functions +require __DIR__ . '/functions.php'; + // autoload boz-mw classes require __DIR__ . '/boz-mw/autoload.php'; diff --git a/includes/boz-mw b/includes/boz-mw index 966771d..91b473a 160000 --- a/includes/boz-mw +++ b/includes/boz-mw @@ -1 +1 @@ -Subproject commit 966771dbbe5d8ec2e6b82da213399ad69c3443d6 +Subproject commit 91b473a17ca561edd83334397d29a7a778dd001a diff --git a/includes/functions.php b/includes/functions.php new file mode 100644 index 0000000..61a92b8 --- /dev/null +++ b/includes/functions.php @@ -0,0 +1,146 @@ +. + +use \cli\Log; +use \cli\Opts; + +/** + * Show the help message and die + */ +function show_help() { + + global $argv; + + echo "Welcome in your MediaWiki Orphanizer bot!\n\n"; + echo "Available options, most of them also on-wiki:\n"; + + // show every available option + foreach( Opts::instance()->getParams() as $param ) { + + // long option + if( $param->hasLongName() ) { + echo " --{$param->getLongName()}"; + if( $param->isValueRequired() ) { + echo "=VALUE"; + } + echo "\n"; + } + + // short option + if( $param->hasShortName() ) { + echo " -{$param->getShortName()}"; + if( $param->isValueRequired() ) { + echo "=VALUE"; + } + echo "\n"; + } + + // description + echo " \t{$param->getDescription()}\n"; + } + + echo "\n Example:\n" . + " \t{$argv[0]} --wiki=itwiki --list=Wikipedia:PDC/Elenco\n\n" . + " Have fun! by Valerio Bozzolan, Daimona Eaytoy\n" ; + + exit( 0 ); + +} + +/** + * Get an option value + * + * It's retrieved from a command line argument, + * or from the on-wiki configuration page, + * or from the default value provided. + * + * @param $name string + * @param $default mixed Default value + */ +function option( $name, $default = null ) { + + // try the wiki option, then the $default + $value = wiki_option( $name, $default ); + + // try the cli option, then wiki option + return Opts::instance()->getArg( $name, $value ); +} + +/** + * Get an on-wiki option value + * + * The option is retrieved from a wiki page + * with a JSON content model. + * + * @param $name string + * @param $default mixed Default value + */ +function wiki_option( $name, $default = null ) { + + $config = wiki_config(); + + // retrieve the option if exists + return isset( $config->{ $name } ) + ? $config->{ $name } + : $default; +} + +/** + * Load the wiki configuration (once) + * + * @return object + */ +function wiki_config() { + static $config = null; + + // retrieve the configuration only once + if( $config === null ) { + $page = Opts::instance()->getArg( 'cfg', 'Utente:OrfanizzaBot/Configurazione' ); + $config = fetch_json_page( $page ); + } + + return $config; +} + +/** + * Fetch a MediaWiki page with the JSON + * content model. + * + * @param $pagename string + * @return object + */ +function fetch_json_page( $pagename ) { + + global $wiki; + + Log::info( "reading $pagename" ); + + $response = + $wiki->fetch( [ + 'action' => 'query', + 'titles' => $pagename, + 'prop' => 'revisions', + 'rvslots' => 'main', + 'rvprop' => 'content', + ] ); + + $revision = reset( $response->query->pages )->revisions[0]; + if ( $revision->slots->main->contentmodel !== 'json' ) { + Log::error( "the page $pagename must have JSON content model" ); + exit( 1 ); + } + + return json_decode( $revision->slots->main->{ '*' } ); +} diff --git a/orphanize.php b/orphanize.php index 8c83065..eeea621 100755 --- a/orphanize.php +++ b/orphanize.php @@ -1,313 +1,352 @@ #!/usr/bin/php . namespace itwikidelbot; // die on whatever error set_error_handler( function( $errno, $errstr, $errfile, $errline ) { if( error_reporting() !== 0 ) { throw new \ErrorException( $errstr, 0, $errno, $errfile, $errline ); } } ); // do not expose from web isset( $argv[ 0 ] ) or exit( 1 ); // autoload classes require __DIR__ . '/includes/autoload.php'; // require config file file_exists( $config_path = __DIR__ . '/config.php' ) or die( "Please provide the configuration file at $config_path\n" ); require $config_path; -// allowed options -$opts = getopt( 'h', [ - 'wiki:', - 'list:', - 'summary:', - 'help' -] ); - -// show help -if( isset( $opts[ 'h' ] ) || isset( $opts[ 'help' ] ) ) { - echo "Welcome in your MediaWiki Orphanizer bot!\n\n" . - " Usage: {$argv[0]} [OPTIONS]\n" . - " Options: --wiki UID Specify a wiki from it's UID.\n" . - " --list PAGENAME Specify a pagename that should\n" . - " contain the wikilinks to be\n" . - " orphanized by this bot.\n" . - " --cfg PAGENAME Read the config from the specified\n" . - " wikipage" . - " --help Show this message and quit.\n" . - " Example:\n" . - " {$argv[0]} --wiki itwiki --list Wikipedia:PDC/Elenco\n\n" . - " Have fun! by Valerio Bozzolan\n" ; - exit( 1 ); -} - -// title source -$TITLE_SOURCE = - isset( $opts[ 'list' ] ) - ? $opts[ 'list' ] - : 'Utente:.avgas/Wikilink da orfanizzare'; - -// cfg page -$CFG_PAGE = - isset( $opts[ 'cfg' ] ) - ? $opts[ 'cfg' ] - : 'Utente:OrfanizzaBot/Configurazione'; - // how much titles at time requesting - this is a MediaWiki limit define( 'MAX_TRANCHE_TITLES', 50 ); // classes used -use \cli\Log; use \cli\Input; +use \cli\Opts; +use \cli\ParamFlag; +use \cli\ParamValued; +use \cli\Log; use \web\MediaWikis; use \mw\Wikilink; use \mw\Ns; use \mw\API\ProtectedPageException; +use \regex\Generic as Regex; + +// register available options +$opts = Opts::instance()->register( [ + // register arguments with a value + new ParamValued( 'wiki', null, 'Specify a wiki from it\'s UID' ), + new ParamValued( 'cfg', null, 'Title of an on-wiki configuration page with JSON content model' ), + new ParamValued( 'list', null, 'Specify a pagename that should contain the wikilinks to be orphanized' ), + new ParamValued( 'summary', null, 'Edit summary' ), + new ParamValued( 'ns', null, 'Namespace whitelist' ), + new ParamValued( 'delay', null, 'Additional delay between each edit' ), + new ParamValued( 'warmup', null, 'Start only if the last edit on the list was done at least $warmup seconds ago' ), + new ParamValued( 'cooldown', null, 'End early when reaching this number of edits' ), + new ParamValued( 'seealso', null, 'Title of your local "See also" section' ), + + // register arguments without a value + new ParamFlag( 'debug', null, 'Increase verbosity' ), + new ParamFlag( 'help', 'h', 'Show this message and quit' ), + new ParamFlag( 'no-interaction', null, 'Do not confirm every change' ), +] ); -// wiki identifier -$wiki_uid = - isset( $opts[ 'wiki' ] ) - ? $opts[ 'wiki' ] - : 'itwiki'; +// show help screen +if( $opts->getArg( 'help' ) ) { + show_help(); +} -// wiki instance -$wiki = Mediawikis::findFromUid( $wiki_uid ); +// cli-only parameters +$NO_INTERACTION = $opts->getArg( 'no-interaction' ); +$TITLE_SOURCE = $opts->getArg( 'list', 'Utente:.avgas/Wikilink da orfanizzare' ); -Log::info( "reading $CFG_PAGE" ); -$cfgRevs = - $wiki->fetch( [ - 'action' => 'query', - 'titles' => $CFG_PAGE, - 'prop' => 'revisions', - 'rvslots' => 'main', - 'rvprop' => 'content', - ] ); +Log::info( "start" ); -$cfgRev = reset( $cfgRevs->query->pages )->revisions[0]; -if ( $cfgRev->slots->main->contentmodel !== 'json' ) { - Log::error( 'The cfg page must have JSON content model.' ); - exit( 1 ); +// increase verbosity +if( $opts->getArg( 'debug' ) ) { + Log::$DEBUG = true; } -$cfg = json_decode( $cfgRev->slots->main->{ '*' } ); -// edit summary -$SUMMARY = - isset( $cfg->summary ) - ? $cfg->summary - : "Bot TEST: orfanizzazione voci eliminate in seguito a [[WP:RPC|consenso cancellazione]]"; +// wiki instance +$wiki = Mediawikis::findFromUid( $opts->getArg( 'wiki', 'itwiki' ) ); + +// load the wiki config +wiki_config(); + +// parameters available both from cli and on-wiki +$SUMMARY = option( 'summary', "Bot TEST: orfanizzazione voci eliminate in seguito a [[WP:RPC|consenso cancellazione]]" ); +$NS = option( 'ns' ); +$WARMUP = option( 'warmup', -1 ); +$COOLDOWN = option( 'cooldown', 1000 ); +$DELAY = option( 'delay', 0 ); +$SEEALSO = option( 'seealso', 'Voci correlate' ); + +// query titles to be orphanized alongside the last revision of the list +$responses = + $wiki->createQuery( [ + 'action' => 'query', + 'titles' => $TITLE_SOURCE, + 'prop' => [ + 'links', + 'revisions', + ], + 'rvslots' => 'main', + 'rvprop' => 'timestamp', + ] ); -// limit to a certain namespace (default is every namespace) -$NS = - isset( $cfg->ns ) - ? $cfg->ns - : null; +// collect links and take the last edit timestamp +$titles_to_be_orphanized = []; -// query last revision Log::info( "reading $TITLE_SOURCE" ); -$links = - $wiki->fetch( [ - 'action' => 'query', - 'prop' => 'links', - 'titles' => $TITLE_SOURCE - ] ); +foreach( $responses as $response ) { + foreach( $response->query->pages as $page ) { + + // check warmup + $timestamp = reset( $page->revisions )->timestamp; + $timestamp = \DateTime::createFromFormat( \DateTime::ISO8601, $timestamp ); + $seconds = time() - $timestamp->getTimestamp(); + if( $seconds < $WARMUP ) { + Log::info( "edited just $seconds seconds ago: quit until warmup $WARMUP" ); + exit( 1 ); + } -$titles_to_be_orphanized = isset( reset( $links->query->pages )->links ) - ? reset( $links->query->pages )->links - : null; -if ( $titles_to_be_orphanized === null ) { - Log::error( 'check links list' ); - exit( 1 ); -} elseif ( count( $titles_to_be_orphanized ) === 0 ) { - Log::error( 'wtf' ); - exit( 1 ); + // collect links + foreach( $page->links as $link ) { + $titles_to_be_orphanized[] = Ns::defaultCanonicalName( $link->ns ) . $link->title; + } + } } -$titles_to_be_orphanized = array_map( - function( $t ) { - return Ns::defaultCanonicalName( $t->ns ) . $t->title; - }, - $titles_to_be_orphanized -); +// die if no links +if( ! $titles_to_be_orphanized ) { + Log::info( 'empty list' ); + exit( 1 ); +} // keep a copy $involved_pagetitles = $titles_to_be_orphanized; // log titles Log::info( 'read ' . count( $titles_to_be_orphanized ) . ' pages to be orphanized:' ); foreach( $titles_to_be_orphanized as $title ) { Log::info( " $title" ); } // associative array of page IDs as key and a boolean as value containg pages to be orphanized $involved_pageids = []; // note that the API accepts a maximum trance of titles while( $less_titles_to_be_orphanized = array_splice( $titles_to_be_orphanized, 0, MAX_TRANCHE_TITLES ) ) { - // API argumento for the linksto query + // API arguments for the linkshere query $linksto_args = [ - 'action' => 'query', - 'titles' => $less_titles_to_be_orphanized, - 'prop' => 'linkshere', - 'lhprop' => 'pageid', - 'lhlimit' => 300, + 'action' => 'query', + 'titles' => $less_titles_to_be_orphanized, + 'prop' => 'linkshere', + 'lhprop' => 'pageid', + 'lhshow' => '!redirect', + 'lhlimit' => 300, ]; // limit to certain namespaces if( $NS !== null ) { $linksto_args[ 'lhnamespace' ] = implode( '|', $NS ); } // cumulate the linkshere page ids Log::info( "requesting linkshere..." ); $linksto = $wiki->createQuery( $linksto_args ); foreach( $linksto as $response ) { foreach( $response->query->pages as $page ) { if( isset( $page->linkshere ) ) { foreach( $page->linkshere as $linkingpage ) { $involved_pageids[] = (int) $linkingpage->pageid; } } } } } // count of involved pages Log::info( sprintf( "found %d pages containing the %d involved wlinks", count( $involved_pageids ), count( $involved_pagetitles ) ) ); +// number of edited pages +$edits = 0; + // note that the API accepts a maximum tranche of IDs while( $less_involved_pageids = array_splice( $involved_pageids, 0, MAX_TRANCHE_TITLES ) ) { // query last revision $responses = $wiki->createQuery( [ 'action' => 'query', 'pageids' => $less_involved_pageids, 'prop' => 'revisions', 'rvslots' => 'main', 'rvprop' => [ 'content', 'timestamp', ], ] ); // for each response foreach( $responses as $response ) { // for each page foreach( $response->query->pages as $page ) { + // avoid too many edits + if( $edits > $COOLDOWN ) { + Log::info( "reached cooldown: stop" ); + exit( 0 ); + } + // page ID to be edited $pageid = $page->pageid; // does it have a revision? if( !isset( $page->revisions[ 0 ] ) ) { continue; } // the first revision $revision = $page->revisions[ 0 ]; // timestamp of the revision useful to avoid edit conflicts $timestamp = $revision->timestamp; // wikitext from the main slot of this revision $wikitext_raw = $revision->slots->main->{ '*' }; // create a Wikitext object $wikitext = $wiki->createWikitext( $wikitext_raw ); // for each of the titles to be orphanized foreach( $involved_pagetitles as $involved_pagetitle ) { // parse the orphanizing title $title = $wiki->createTitleParsing( $involved_pagetitle ); // a wikilink with and without alias $wikilink_simple = $wiki->createWikilink( $title, Wikilink::NO_ALIAS ); $wikilink_alias = $wiki->createWikilink( $title, Wikilink::WHATEVER_ALIAS ); // replace simple links e.g. [[Hello]] $wikilink_regex_simple = $wikilink_simple->getRegex( [ 'title-group-name' => 'title', ] ); // replace links with alias e.g. [[Hello|whatever]] $wikilink_regex_alias = $wikilink_alias->getRegex( [ 'alias-group-name' => 'alias', ] ); + // replace entry from "See also" section + $wikilink_regex_clean = $wikilink_simple->getRegex(); + $wikilink_regex_clean = Regex::spaceBurger( $wikilink_regex_clean ); + $seealso = preg_quote( $SEEALSO ); + $seealso_regex = + '/' . + Regex::groupNamed( "\\n== *$seealso *== *((?!=).*\n)*", 'keep' ) . + Regex::groupNamed( "[ \\t]*\*[ \\t]*{$wikilink_regex_clean}.*\\n", 'wlink' ) . + '/'; + Log::debug( "regex simple wikilink:" ); Log::debug( $wikilink_regex_simple ); Log::debug( "regex wikilink aliased:" ); Log::debug( $wikilink_regex_alias ); + Log::debug( "regex see also:" ); + Log::debug( $seealso_regex ); + + // strip out the entry from «See also» section + $wikitext->pregReplaceCallback( $seealso_regex, function ( $matches ) { + return $matches[ 'keep' ]; + } ); + // convert '[[Hello]]' to 'Hello' $wikitext->pregReplaceCallback( "/$wikilink_regex_simple/", function ( $matches ) { return $matches[ 'title' ]; } ); // convert '[[Hello|world]]' to 'world' $wikitext->pregReplaceCallback( "/$wikilink_regex_alias/", function ( $matches ) { return $matches[ 'alias' ]; } ); } // end loop titles to be orphanized // check for changes and save if( $wikitext->isChanged() ) { Log::info( "changes on page $pageid:" ); foreach( $wikitext->getHumanUniqueSobstitutions() as $sobstitution ) { Log::info( "\t $sobstitution" ); } - if( 'n' !== Input::yesNoQuestion( "confirm changes" ) ) { + + if( $NO_INTERACTION || 'n' !== Input::yesNoQuestion( "confirm changes" ) ) { try { + + // the entire world absolutely needs this shitty ASCII animation - trust me + if( $edits && $DELAY ) { + Log::info( "delay $DELAY seconds", [ 'newline' => false ] ); + for( $i = 0; $i < $DELAY; $i++ ) { + sleep( 1 ); + echo '.'; + } + echo "\n"; + } + + // eventually login and save $wiki->login()->edit( [ - 'pageid' => $pageid, - 'text' => $wikitext->getWikitext(), - 'summary' => $SUMMARY, - 'timestamp' => $timestamp, - 'minor' => 1, - 'bot' => 1, + 'pageid' => $pageid, + 'text' => $wikitext->getWikitext(), + 'summary' => $SUMMARY, + 'basetimestamp' => $timestamp, + 'minor' => 1, + 'bot' => 1, ] ); + + $edits++; + } catch( ProtectedPageException $e ) { Log::warn( "skip protected page $pageid" ); } } // end confirmation } // end save } // end loop pages } // end loop responses } // end loop involved page IDs + +Log::info( "end" );