diff --git a/tools/mega-export-csv.php b/tools/mega-export-csv.php index dcc48c9..5de5808 100755 --- a/tools/mega-export-csv.php +++ b/tools/mega-export-csv.php @@ -1,204 +1,204 @@ -#!/usr/bin/php +#!/usr/bin/env php . // exit if not CLI $argv or exit( 1 ); // load boz-mw require __DIR__ . '/../autoload-with-laser-cannon.php'; define( 'MEDIAWIKI_DATE_FORMAT', 'Y-m-d\TH:i:sZ'); // the number '500' gives to much $DEFAULT_LIMIT = 100; // load configuration config_wizard( 'config.php' ); use \web\MediaWikis; use \cli\Log; // all the available wiki UIDs $mediawiki_uids = implode( ', ', MediaWikis::allUIDs() ); // register all CLI parameters $opts = cli_options() ->addValued( 'wiki', null, "Available wikis: $mediawiki_uids" ) ->addValued( 'limit', null, "Number of revisions for each request", $DEFAULT_LIMIT ) ->addValued( 'file', null, "Output filename", 'export.xml' ) ->addFlag( 'help', 'h', "Show this help and quit" ); $messages = []; // choosen wiki $wiki_uid = $opts->get( 'wiki' ); if( !$wiki_uid ) { $messages[] = "Please specify --wiki=WIKI"; } // page titles $page_titles = $opts::unnamedArguments(); if( !$page_titles ) { $messages[] = "Please specify some page titles"; } // output filename $filename = $opts->get( 'file' ); if( !$filename ) { // as default assume standard output $filename = 'php://output'; } $limit = (int) $opts->get( 'limit' ); // show the help $show_help = $opts->get( 'help' ); if( $show_help ) { $messages = []; } else { $show_help = $messages; } if( $show_help ) { echo "Usage:\n {$argv[ 0 ]} --wiki=WIKI --file=export.xml [OPTIONS] Page_title\n"; echo "Allowed OPTIONS:\n"; $opts->printParams(); foreach( $messages as $msg ) { echo "\nError: $msg"; } echo "\n"; exit( $opts->get( 'help' ) ? 0 : 1 ); } // try to open the file $file = fopen( $filename, 'w' ); if( !$file ) { Log::error( "Can't open file '$filename'" ); exit( 1 ); } // pick the wiki $wiki = wiki( $wiki_uid ); if(!$wiki) { echo "Unrecognized wiki\n"; exit( 2 ); } // login (if you are a bot you have smaller limitations) $wiki->login(); // build the MediaWiki API query $requests = $wiki->createQuery( [ 'action' => 'query', 'titles' => $page_titles, 'prop' => 'revisions', 'rvprop' => [ 'ids', 'flags', 'timestamp', 'user', 'userid', 'size', 'slotsize', 'sha1', 'comment', ], 'rvslots' => 'main', 'rvlimit' => $limit, ] ); // total number of revisions $total = 0; $n_requests = 1; fputcsv( $file, [ "Date", "Time", "Dimension", "User ID", "User name", "Comment", ] ); // do not print to the out foreach( $requests as $request ) { // show a kind of progress Log::info( sprintf( "processing request %d (continuing from $total revisions)", $n_requests++ ) ); $response_warning_shown = false; foreach( $request->query->pages as $page ) { if( isset( $page->missing ) ) { Log::error( "Page '{$page->title}' is missing" ); exit( 1 ); } $alert_much_revisions = true; foreach( $page->revisions as $i => $revision ) { // avoid nonsense revisions if( empty( $revision->comment ) ) { $count = count( $page->revisions ); if( $count !== $limit && !$response_warning_shown ) { Log::warn( "response with $count revisions instead of $limit: consider to lower your limit (ignore if you see this just once)" ); $response_warning_shown = true; } } $total++; foreach( $revision->slots as $slot ) { $safe_user = $revision->user; $safe_userid = $revision->userid; $size = $revision->size; $timestamp = $revision->timestamp; $comment = $revision->comment; $timestamp_date = DateTime::createFromFormat( MEDIAWIKI_DATE_FORMAT, $timestamp ); $timestamp_ymd = $timestamp_date->format( 'Y-m-d' ); $timestamp_his = $timestamp_date->format( 'H:i:s' ); fputcsv( $file, [ $timestamp_ymd, $timestamp_his, $size, $safe_user, $safe_userid, $comment, ] ); } } } } Log::info( sprintf( "you mega-exported %d revisions!", $total ) ); fclose( $file ); diff --git a/tools/mega-export-xml.php b/tools/mega-export-xml.php index 45069b5..fccf955 100755 --- a/tools/mega-export-xml.php +++ b/tools/mega-export-xml.php @@ -1,200 +1,200 @@ -#!/usr/bin/php +#!/usr/bin/env php . // exit if not CLI $argv or exit( 1 ); // load boz-mw require __DIR__ . '/../autoload-with-laser-cannon.php'; // the number '500' gives to much $DEFAULT_LIMIT = 100; // load configuration config_wizard( 'config.php' ); use \web\MediaWikis; use \cli\Log; // all the available wiki UIDs $mediawiki_uids = implode( ', ', MediaWikis::allUIDs() ); // register all CLI parameters $opts = cli_options() ->addValued( 'wiki', null, "Available wikis: $mediawiki_uids" ) ->addValued( 'limit', null, "Number of revisions for each request", $DEFAULT_LIMIT ) ->addValued( 'file', null, "Output filename", 'export.xml' ) ->addFlag( 'help', 'h', "Show this help and quit" ); $messages = []; // choosen wiki $wiki_uid = $opts->get( 'wiki' ); if( !$wiki_uid ) { $messages[] = "Please specify --wiki=WIKI"; } // page titles $page_titles = $opts::unnamedArguments(); if( !$page_titles ) { $messages[] = "Please specify some page titles"; } // output filename $filename = $opts->get( 'file' ); if( !$filename ) { $messages[] = "Please specify a filename"; } $limit = (int) $opts->get( 'limit' ); // show the help $show_help = $opts->get( 'help' ); if( $show_help ) { $messages = []; } else { $show_help = $messages; } if( $show_help ) { echo "Usage:\n {$argv[ 0 ]} --wiki=WIKI --file=export.xml [OPTIONS] Page_title\n"; echo "Allowed OPTIONS:\n"; $opts->printParams(); foreach( $messages as $msg ) { echo "\nError: $msg"; } echo "\n"; exit( $opts->get( 'help' ) ? 0 : 1 ); } // try to open the file $file = fopen( $filename, 'w' ); if( !$file ) { Log::error( "Can't open file '$filename'" ); exit( 1 ); } // pick the wiki and login $wiki = wiki( $wiki_uid )->login(); // build the MediaWiki API query $requests = $wiki->createQuery( [ 'action' => 'query', 'titles' => $page_titles, 'prop' => 'revisions', 'rvprop' => [ 'ids', 'flags', 'timestamp', 'user', 'userid', 'size', 'slotsize', 'sha1', 'comment', 'content', ], 'rvslots' => 'main', 'rvlimit' => $limit, ] ); // total number of revisions $total = 0; $n_requests = 1; // do not print to the out $out = '' . "\n"; foreach( $requests as $request ) { // show a kind of progress Log::info( sprintf( "processing request %d (continuing from $total revisions)", $n_requests++ ) ); $response_warning_shown = false; foreach( $request->query->pages as $page ) { if( isset( $page->missing ) ) { Log::error( "Page '{$page->title}' is missing" ); exit( 1 ); } $alert_much_revisions = true; foreach( $page->revisions as $i => $revision ) { // avoid nonsense revisions if( empty( $revision->comment ) ) { $count = count( $page->revisions ); if( $count !== $limit && !$response_warning_shown ) { Log::warn( "response with $count revisions instead of $limit: consider to lower your limit (ignore if you see this just once)" ); $response_warning_shown = true; } } $total++; foreach( $revision->slots as $slot ) { // avoid nonsense slots if( empty( $slot->contentmodel ) ) { continue; } $safe_user = htmlentities( $revision->user ); $safe_userid = htmlentities( $revision->userid ); $safe_comment = htmlentities( $revision->comment ); $safe_model = htmlentities( $slot->contentmodel ); $safe_format = htmlentities( $slot->contentformat ); $safe_text = htmlentities( $slot->{'*'} ); $out .= "\n"; $out .= "\t{$revision->revid}\n"; $out .= "\t{$revision->parentid}\n"; $out .= "\t{$revision->timestamp}\n"; $out .= "\t\n"; $out .= "\t\t$safe_user\n"; $out .= "\t\t$safe_userid\n"; $out .= "\t\n"; $out .= "\t$safe_comment ?>"; $out .= "\t$safe_model\n"; $out .= "\t$safe_format\n"; $out .= "\tsize}\">$safe_text\n"; $out .= "\t{$revision->sha1}\n"; $out .= "\n"; } } } // write the file in chunks fwrite( $file, $out ); $out = ''; } Log::info( sprintf( "you mega-exported $total revisions! nice shot! See %s", $opts->get( 'file' ) ) ); fwrite( $file, "\n" ); fclose( $file ); diff --git a/tools/replace.php b/tools/replace.php index 10284b5..b2dcdf7 100755 --- a/tools/replace.php +++ b/tools/replace.php @@ -1,262 +1,262 @@ -#!/usr/bin/php +#!/usr/bin/env php . // exit if not CLI $argv or exit( 1 ); // default seconds to wait in always mode $DEFAULT_ALWAYS_WAIT = 3; // load boz-mw require __DIR__ . DIRECTORY_SEPARATOR . '..' . DIRECTORY_SEPARATOR . 'autoload-with-laser-cannon.php'; // load configuration config_wizard( 'config.php' ); use \cli\Log; use \cli\Input; use \cli\Opts; use \cli\Param; use \cli\ParamFlag; use \cli\ParamFlagLong; use \cli\ParamValued; use \cli\ParamValuedLong; use \web\MediaWikis; // all the available wiki UIDs $mediawiki_uids = []; foreach( MediaWikis::all() as $site ) { $mediawiki_uids[] = $site::UID; } $mediawiki_uids = implode( ', ', $mediawiki_uids ); // these classes are useful only to distinguish this type of params :^) class APIParam extends ParamValuedLong {} class APIParamSub extends APIParam {} // register all CLI parameters $opts = new Opts( [ new ParamValuedLong( 'wiki', "Available wikis: $mediawiki_uids" ), new APIParam( 'generator', 'Choose between: linkshere, categorymembers, transcludedin, search' ), new APIParamSub( 'glhnamespace', 'Only in linkshere: Namespace number' ), new APIParamSub( 'gtinamespace', ' transcludedin: Namespace number' ), new APIParamSub( 'gcmtitle', ' Category name prefixed' ), new APIParamSub( 'gcmpageid', ' Category page ID' ), new APIParamSub( 'gcmnamespace', ' categorymembers: Namespace number' ), new APIParamSub( 'gsrsearch', ' search: Search term' ), new APIParamSub( 'gsrnamespace', ' Namespace number' ), new APIParam( 'titles', 'Title of pages to work on' ), new APIParam( 'pageids', 'Page IDs to work on' ), new ParamFlagLong( 'plain', 'Use plain text instead of regexes (default)' ), new ParamFlagLong( 'regex', 'Use regexes instead of plain text' ), new ParamValued( 'summary', 'm', 'Specify an edit summary' ), new ParamValuedLong( 'limit', 'Maximum number of replacements for each SEARCH' ), new ParamFlagLong( 'not-minor', 'Do not mark this change as minor' ), new ParamFlagLong( 'not-bot', 'Do not mark this change as edited by a bot' ), new ParamValuedLong( 'rvsection', 'Number of section to be edited' ), new ParamFlagLong( 'always', 'Always run without asking y/n' ), new ParamValuedLong( 'always-wait', "Seconds to wait during --always mode (default $DEFAULT_ALWAYS_WAIT)" ), new ParamFlagLong( 'simulate', 'Show changes without saving' ), new ParamFlagLong( 'show', 'Show the wikitext before saving' ), new ParamFlag( 'help', 'h', 'Show this help and quit' ), ] ); // unnamed arguments $main_args = Opts::unnamedArguments(); // wait seconds for --always $ALWAYS_WAIT = (int) $opts->getArg( 'always-wait', $DEFAULT_ALWAYS_WAIT ); // operate using regex or in plain text? $IS_PLAIN = $opts->getArg( 'plain', true ); $IS_REGEX = $opts->getArg( 'regex', false ); // maximum number of replacements for each LIMIT $LIMIT = (int) $opts->getArg( 'limit', -1 ); // show the help? $help = (bool) $opts->getArg( 'help' ); // the wiki is mandatory if( ! $help && ! $opts->getArg( 'wiki' ) ) { $help = "Please specify a --wiki=VALUE"; } // pick the wiki $wiki = MediaWikis::findFromUID( $opts->getArg( 'wiki' ) ); if( ! $help && ! $wiki ) { $help = "Please specify a valid wiki UID from: $mediawiki_uids"; } // the generator is mandatory if( ! $help && ! $opts->getArg( 'generator' ) ) { $help = "Please specify the --generator=VALUE"; } // the search and replace are mandatory if( ! $help && ! $main_args ) { $help = "Please specify a SEARCH and a REPLACE"; } // every search must have its replace if( ! $help && count( $main_args ) % 2 !== 0 ) { $last = $main_args[ count( $main_args ) - 1 ]; $help = "Please specify a REPLACE for your latest SEARCH ($last)"; } // plain vs regex if( ! $help && null === $IS_REGEX ) { $help = "Please specify if it's --plain or it's --regex"; } // plain with regex if( ! $help && $opts->getArg( 'regex' ) && $opts->getArg( 'plain' ) ) { $help = "Please do not specify both --regex or --plain"; } // show the help if( $help ) { echo "Usage:\n {$argv[ 0 ]} [OPTIONS] SEARCH... REPLACE...\n"; echo "Allowed OPTIONS:\n"; $opts->printParams(); echo "Example:\n {$argv[ 0 ]} --wiki=itwiki --generator=allpages 'a' 'afa'\n"; if( is_string( $help ) ) { echo "\nError: $help\n"; } exit( $opts->getArg( 'help' ) ? 0 : 1 ); } // API arguments $args = [ 'action' => 'query', 'prop' => 'revisions', 'rvslots' => 'main', 'rvprop' => [ 'content', 'timestamp', ] ]; // apply API arguments from CLI API parameters foreach( $opts->getParams() as $param ) { if( $param instanceof APIParam ) { $value = $param->getValue(); if( $value ) { $args[ $param->getLongName() ] = $value; } } } $wiki->login(); // do the API query with continuation support $results = $wiki->createQuery( $args ); foreach( $results->getGenerator() as $response ) { if( ! isset( $response->query ) ) { Log::error( 'empty API output (have you specified nothing?) try --help' ); exit( 1 ); } foreach( $response->query->pages as $page ) { // show the page title Log::info( "Page [[{$page->title}]]" ); // populate the slot with backward compatibility to MediaWiki <= 1.32 $main_slot = $page->revisions[ 0 ]; if( isset( $main_slot->slots->main ) ) { $main_slot = $main_slot->slots->main; } // Wikitext object $wikitext = $wiki->createWikitext( $main_slot->{ '*' } ); // apply search and replacements $n = count( $main_args ); for( $i = 0; $i < $n; $i += 2 ) { $search = $main_args[ $i ]; $replace = $main_args[ $i + 1 ]; if( $IS_REGEX ) { $wikitext->pregReplace( $search, $replace, $LIMIT ); } else { $wikitext->strReplace( $search, $replace, $LIMIT ); } } // eventually show the wikitext if( $opts->getArg( 'show' ) ) { Log::info( "Wikitext:\n{$wikitext->getWikitext()}" ); } // are there changes? if( $wikitext->getSobstitutions() ) { // show changes Log::info( 'Changes:' ); foreach( $wikitext->getUniqueSobstitutions() as $change ) { list( $a, $b, $n ) = $change; Log::info( sprintf( '% 40s → % 40s (%d times)', $a, $b, $n ) ); } // edit summary $summary = $opts->getArg( 'summary', 'Bot: changed ' . implode( ', ', $wikitext->getHumanUniqueSobstitutions() ) ); // show the summary if it's auto-generated if( ! $opts->getArg( 'summary' ) ) { Log::info( "Summary: \t $summary" ); } if( ! $opts->getArg( 'simulate' ) ) { $proceed = true; if( $opts->getArg( 'always' ) && $ALWAYS_WAIT ) { Log::info( "Waiting $ALWAYS_WAIT seconds before saving", [ 'newline' => false ] ); for( $i = 0; $i < $ALWAYS_WAIT; $i++ ) { echo "."; sleep( 1 ); } echo "\n"; } else { $proceed = 'y' === Input::yesNoQuestion( "Save page [[{$page->title}]]?" ); } if( $proceed ) { // save $wiki->edit( [ 'pageid' => $page->pageid, 'basetimestamp' => $page->revisions[ 0 ]->timestamp, 'text' => $wikitext->getWikitext(), 'section' => $opts->getArg( 'rvsection' ), 'summary' => $summary, 'minor' => ! $opts->getArg( 'not-minor' ), 'bot' => ! $opts->getArg( 'not-bot' ), ] ); Log::info( 'saved' ); } else { Log::info( 'not saved' ); } } } else { Log::info( 'no changes' ); } } } diff --git a/tools/wikidata-review-contributor.php b/tools/wikidata-review-contributor.php index 7717056..9c9fb6a 100755 --- a/tools/wikidata-review-contributor.php +++ b/tools/wikidata-review-contributor.php @@ -1,201 +1,201 @@ -#!/usr/bin/php +#!/usr/bin/env php . // exit if not CLI $argv or exit( 1 ); // load the framework require __DIR__ . '/../autoload.php'; // used classes from boz-mw use \cli\Log; use \cli\Input; use \cli\Opts; use \wm\Wikidata; use \mw\API\PageMatcher; use \cli\ConfigWizard; use \cli\ParamFlag; // load configuration or create it ConfigWizard::requireOrCreate( __DIR__ . '/config.php' ); // register all CLI parameters $opts = new Opts( [ new ParamFlag( 'help', 'h', "Show this help and quit" ), new ParamFlag( 'interactive', 'i', "Enter in interactive mode" ), new ParamFlag( 'nologin', 'n', "If you do not want to login" ), ] ); // check if the software is interactive or not $INTERACTIVE = $opts->getArg( 'interactive' ); // check if we want to login $NOLOGIN = $opts->getArg( 'nologin' ); // command line arguments $arguments = Opts::unnamedArguments(); if( count( $arguments ) < 2 ) { help( [ "Please specify both the SPARQL file and the user name to blame" ] ); } // take the SPARQL file pathname $sparql_file = array_shift( $arguments ); // user names to blame $user_names = $arguments; // no file no party if( !file_exists( $sparql_file ) ) { help( [ sprintf( "The file '%s' does not exist", $sparql_file ) ] ); } // read the SPARQL query $sparql_query = file_get_contents( $sparql_file ); if( !$sparql_query ) { help( [ sprintf( "Obtained a non-empty SPARQL query from the file '%s'", $sparql_file ) ] ); } // show the help $show_help = $opts->getArg( 'help' ); if( $show_help ) { help(); } // activate Wikidata power and login $wikidata = Wikidata::instance(); // login to download more infos if( !$NOLOGIN ) { $wikidata->login(); } // an array of item IDs $ids = []; // some stats $founds = 0; $allcontribs = 0; $lastcontribdate = ''; // exec the query and store the item IDs $rows = $wikidata->querySPARQL( $sparql_query ); foreach( $rows as $row ) { $item = $row->item ?? null; // no item no party if( !$item ) { Log::error( "the query does not return any 'item' column - probably you forgot to SELECT the '?item'" ); exit( 2 ); } $ids[] = basename( $item->value ); } Log::info( "Starting looking for contributions" ); // and now the magic begins - looks for usercontribs // https://www.wikidata.org/w/api.php?action=help&modules=query%2Busercontribs $requests = $wikidata->createQuery( [ 'action' => 'query', 'list' => 'usercontribs', 'ucuser' => $user_names, 'ucprop' => ['ids', 'title', 'timestamp'], 'ucdir' => 'newer', 'uclimit' => 'max', ] ); foreach( $requests as $request ) { // loop the contributions $contribs = $request->query->usercontribs; foreach( $contribs as $contrib ) { $timestamp = $contrib->timestamp; $revid = $contrib->revid; // check if this is a known page if( in_array( $contrib->title, $ids, true ) ) { Log::info( "$timestamp - https://www.wikidata.org/w/index.php?diff=$revid" ); $founds++; // allow to pause the script if( $INTERACTIVE ) { Input::askInput( "(Press ENTER)" ); } } $allcontribs++; $lastcontribdate = $contrib->timestamp; } Log::info( "Last date: $lastcontribdate (read $allcontribs contribs)" ); } // riepilogue Log::info( "RIEPILOGUE" ); Log::info( " Read contributions: $allcontribs" ); Log::info( " Last contribution read: $lastcontribdate" ); Log::info( " Total matches: $founds" ); if( !$founds ) { Log::error( "Sorry, no results found!" ); } /** * Print an help message * * @param array $errors */ function help( $errors = [] ) { global $argv, $opts; echo "Usage:\n {$argv[ 0 ]} FILE.sparql USER_NAME\n"; echo "Allowed OPTIONS:\n"; foreach( $opts->getParams() as $param ) { $commands = []; if( $param->hasLongName() ) { $commands[] = '--' . $param->getLongName(); } if( $param->hasShortName() ) { $commands[] = '-' . $param->getShortName(); } $command = implode( ' ', $commands ); if( $command && ! $param->isFlag() ) { $command .= $param->isValueOptional() ? '=[VALUE]' : '=VALUE'; } printf( ' % -20s ', $command ); if( $param->hasDescription() ) { echo ' ' . $param->getDescription(); } echo "\n"; } foreach( $errors as $msg ) { echo "\nError: $msg"; } echo "\n"; // quit the program exit( $errors ? 1 : 0 ); } diff --git a/tools/wizard.php b/tools/wizard.php index 171c78a..2a035b2 100755 --- a/tools/wizard.php +++ b/tools/wizard.php @@ -1,31 +1,31 @@ -#!/usr/bin/php +#!/usr/bin/env php . /** * The purpose of this file is to just run a configuration wizard. */ // exit if not CLI $argv or exit( 1 ); // load boz-mw require __DIR__ . DIRECTORY_SEPARATOR . '..' . DIRECTORY_SEPARATOR . 'autoload-with-laser-cannon.php'; // load configuration or run the wizard config_wizard( 'config.php' ); echo "Well done! You already have your config file. Now use a random tool!\n";