diff --git a/README.md b/README.md index 88f5f04..a2c40f1 100644 --- a/README.md +++ b/README.md @@ -1,99 +1,103 @@ # Italian Wikipedia Deletion Bot This software is an Italian Wikipedia bot. It keeps updated the public deletion logs of Italian Wikipedia pages. ## History of the project The script was originally developed by Mauro762 (aka MauroBot) in 2013 using JavaScript and previously developed in Python {{Citation needed}}. ## Installation sudo apt install git php-cli git clone --recursive https://github.com/valerio-bozzolan/ItalianWikipediaDeletionBot ## Configuration 1. Open the file [`config-example.php`](config-example.php) with a text editor 2. Fill your bot credentials 3. Save-as `config.php` ## Usage Run on today: ./bot.php -Run on the latest 10 days: +Run on the latest 10 days from today: ./bot.php --days=10 Run on a specific date (year-month-day): ./bot.php --from=2018-03-13 Run on a specific date and for 10 days before it: ./bot.php --days=10 --from=2018-03-13 +Quit if someone was running the bot on this date just `5` minutes ago (that is the default value): + + ./bot.php --minutes-ago=5 + Ask before saving and enable verbose mode: ./bot.php --ask --verbose Help: ./bot.php --help ## Update To fetch the updates: git pull --recurse-submodules ## Hacking Most of the behaviours — as the content of the categories, the pages, their edit summaries, etc. — can be changed simply hacking the content of the files from the [/templates](/templates) directory. Trust me, you are able to do it. Start becoming familiar with their structure: This is the place for some documentation, expecially about "placeholders". This is the place for a specific example This is the place for the most important part of this file. This part is what this template will generates. It uses stuff like "$1" or "%1%02d" as generic placeholders. To be honest: everything above the `` line it's pure documentation sugar. It's written to help you. I've spent some minutes on them. Please RTFM. asd A non-traumatic template example can be found [here](templates/CATEGORY_YEAR.content.tpl). It describes the generation of the yearly category. ## Framework You may not want to know this: HTTP connections, MediaWiki APIs (including login and tokens etc.), and other stuff, are handled by the [boz-mw](https://github.com/valerio-bozzolan/boz-mw) framework. ## License Copyright (C) 2018 [Valerio Bozzolan](https://it.wikipedia.org/wiki/Utente:Valerio_Bozzolan) This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more details. You should have received a copy of the GNU Affero General Public License along with this program. If not, see . Original files released under [Creative Commons BY-SA 3.0 International](https://creativecommons.org/licenses/by-sa/3.0/) and [WMF terms](https://wikimediafoundation.org/wiki/Special:MyLanguage/Terms_of_Use/it) by [Mauro742](https://it.wikipedia.org/wiki/Utente:Mauro742)/[MauroBot](https://it.wikipedia.org/wiki/Utente:MauroBot): * https://it.wikipedia.org/wiki/Utente:MauroBot/BotCancellazioni/main.js * https://it.wikipedia.org/wiki/Utente:MauroBot/BotCancellazioni/bot.js * https://it.wikipedia.org/wiki/Utente:MauroBot/BotCancellazioni/category.js * https://it.wikipedia.org/wiki/Utente:MauroBot/BotCancellazioni/dateFunctions.js * https://it.wikipedia.org/wiki/Utente:MauroBot/BotCancellazioni/core.js * https://it.wikipedia.org/wiki/Utente:MauroBot/BotCancellazioni/globals.js * https://it.wikipedia.org/wiki/Utente:MauroBot/BotCancellazioni/gui.js diff --git a/bot.php b/bot.php index 07b1a3d..8c88cbb 100755 --- a/bot.php +++ b/bot.php @@ -1,82 +1,98 @@ #!/usr/bin/php . namespace itwikidelbot; // autoload classes require __DIR__ . DIRECTORY_SEPARATOR . 'includes' . DIRECTORY_SEPARATOR . 'autoload.php'; use cli\Log; // load credentials require __DIR__ . DIRECTORY_SEPARATOR . 'config.php'; // the bot must be in sync with the italian community date_default_timezone_set( 'Europe/Rome' ); // allowed only from command line interface if( ! isset( $argv[ 0 ] ) ) { exit( 1 ); } // command line arguments $opts = getopt( 'h', [ - 'ask', // ask for every edit | default: no - 'days:', // days to be processed | default: only one - 'from:', // date to start from | default: from today - 'help', // help message - 'verbose', // debug mode | default: no + 'ask', + 'days:', + 'from:', + 'help', + 'minutes-ago:', + 'verbose', ] ); // help message if( isset( $opts[ 'help' ] ) || isset( $opts[ 'h' ] ) ) { printf( "usage: %s [OPTIONS]\n\n", $argv[ 0 ] ); - echo " --days=DAYS how many days to be processed (default: 1)\n"; - echo " --from=YYYY-MM-DD starting date (default: today)\n"; - echo " --ask ask before saving\n"; - echo " --verbose verbose mode\n"; - echo " -h --help show this help and exit\n"; + echo " --days=DAYS how many days to be processed (default: 1)\n"; + echo " --from=YYYY-MM-DD starting date (default: today)\n"; + echo " --minutes-ago=N quits if the last edit was below N minutes ago (default: 5)\n"; + echo " you can set N to 0 to disable this startup verification\n"; + echo " --ask ask before saving\n"; + echo " --verbose verbose mode\n"; + echo " -h --help show this help and exit\n"; exit( 0 ); } // days to be processed from now to the past $DAYS = isset( $opts[ 'days' ] ) ? (int) $opts[ 'days' ] : 1; // starting date formatted as 'YYYY-MM-DD' $DATE = isset( $opts[ 'from' ] ) ? $opts[ 'from' ] : 'now'; +// minimum last edit age +$MINUTES = isset( $opts[ 'minutes-ago' ] ) + ? (int) $opts[ 'minutes-ago' ] + : 5; + // ask for every edit if( isset( $opts[ 'ask' ] ) ) { Page::$ASK_BEFORE_SAVING = true; } // verbose mode if( isset( $opts[ 'verbose' ] ) ) { Log::$DEBUG = true; } Log::info( 'start' ); $bot = Bot::createFromString( $DATE ); -for( $i = 0; $i < $DAYS; $i++ ) { - $bot->run()->previousDay(); +if( ! $MINUTES || $bot->isLasteditOlderThanMinutes( $MINUTES ) ) { + for( $i = 0; $i < $DAYS; $i++ ) { + $bot->run()->previousDay(); + } +} else { + // do not insist + Log::info( sprintf( + 'skip: someone running less than %d minutes ago', + $MINUTES + ) ); } Log::info( 'end' ); diff --git a/includes/class-Bot.php b/includes/class-Bot.php index 19363c6..8ca72ed 100644 --- a/includes/class-Bot.php +++ b/includes/class-Bot.php @@ -1,221 +1,263 @@ . namespace itwikidelbot; use cli\Log; use DateTime; use DateInterval; +use Exception; /** * The Bot class helps in running the bot */ class Bot { /** * The date * * @var DateTime */ private $dateTime; /** * A cache of already done dates. * It's an array of booleans like [year][month] = true. * * @var array */ private $cache = []; /** * Construct * * @param $date DateTime */ public function __construct( DateTime $date = null ) { if( ! $date ) { $date = new DateTime(); } $this->setDate( $date ); } /** * Static construct * * @param $date string */ public static function createFromString( $date = 'now' ) { return new self( new DateTime( $date ) ); } /** * Static construct * * @param $y int Year * @param $m int Month 1-12 * @param $d int Day 1-31 */ public static function createFromYearMonthDay( $y, $m, $d ) { return new self( DateTime::createFromFormat( "Y m d", "$y $m $d") ); } /** * Get the date * * @return DateTime */ public function getDate() { return $this->dateTime; } /** * Set the date * * @param $date DateTime * @return self */ public function setDate( DateTime $date ) { $this->dateTime = $date; } /** * Add a day * * @return self */ public function nextDay() { return $this->addDays( 1 ); } /** * Add a day * * @return self */ public function previousDay() { return $this->subDays( 1 ); } /** * Add a certain number of days * * @param $days int * @return self */ public function addDays( $days ) { $this->getDate()->add( new DateInterval( sprintf( 'P%dD', $days ) ) ); return $this; } /** * Subtract a certain number of days * * @param $days int * @return self */ public function subDays( $days ) { $this->getDate()->sub( new DateInterval( sprintf( 'P%dD', $days ) ) ); return $this; } + /** + * Fetch the last bot edit on the next page + * + * @return DateTime|null + */ + public function fetchLastedit() { + $lastedit = null; + try { + $lastedit = PageYearMonthDayPDCsCount::createFromDateTime( $this->getDate() ) + ->fetchLasteditDate(); + + } catch( Exception $e ) { + // Unexisting. OK. + } + return $lastedit; + } + + /** + * Is the last edit date older than some seconds? (on the next page) + * + * @param $seconds int + * @return bool + */ + public function isLasteditOlderThanSeconds( $seconds ) { + $lastedit = $this->fetchLastedit(); + if( ! $lastedit ) { + return true; // Unexisting. OK. + } + return time() - $lastedit->format( 'U' ) > $seconds; + } + + /** + * Is the last edit date older than some minutes? (on the next page) + * + * @param $minutes int + * @return bool + */ + public function isLasteditOlderThanMinutes( $minutes ) { + return $this->isLasteditOlderThanSeconds( 60 * $minutes ); + } + /** * Run the bot at the internal date * * @TODO: do not repeat twice the same yearly and montly categories * @return self */ public function run() { // date initialization $date = $this->getDate(); $year = $date->format( 'Y' ); $month = $date->format( 'n' ); // 1-12 $day = $date->format( 'j' ); // create the yearly category (once) $cache = & $this->cache; if( ! isset( $cache[ $year ] ) ) { $cache[ $year ] = []; ( new CategoryYear( $year ) ) ->saveIfNotExists(); } // create the monthly category (once) $cache = & $cache[ $year ]; if( ! isset( $cache[ $month ] ) ) { $cache[ $month ] = true; ( new CategoryYearMonth( $year, $month ) ) ->saveIfNotExists(); } Log::info( "work on $year/$month/$day" ); // PDCs indexed by page ID $pdcs = []; // handle every PDC type foreach( CategoryYearMonthDayTypes::all() as $CategoryType ) { // fetch PDCs from this type $category_type = new $CategoryType( $year, $month, $day ); $category_type_pdcs = $category_type->fetchPDCs(); // save the specific daily category type only if it's not empty // ...or only if it's the main category (that can be without pages) if( $category_type_pdcs || get_class( $category_type ) === CategoryYearMonthDay::class ) { $category_type->saveIfNotExists(); } // merge the same PDCs into one foreach( $category_type_pdcs as $category_type_pdc ) { $id = $category_type_pdc->getID(); if( isset( $pdcs[ $id ] ) ) { $pdcs[ $id ]->merge( $category_type_pdc ); } else { $pdcs[ $id ] = $category_type_pdc; } } } // discard PDCs that do not belong to this date $pdcs = PDCs::filterNotInDate( $pdcs, $date ); Log::info( sprintf( sprintf( "found %d PDCs", count( $pdcs ) ) ) ); // sort by creation date PDCs::sortByCreationDate( $pdcs ); // index then by their PDC_TYPE $pdcs_by_type = PDCs::indexByType( $pdcs ); // save the counting page PageYearMonthDayPDCsCount::createFromDateTimePDCs( $this->getDate(), $pdcs_by_type ) ->save(); // save the log page PageYearMonthDayPDCsLog::createFromDateTimePDCs( $this->getDate(), $pdcs_by_type ) ->save(); return $this; } } diff --git a/includes/class-Page.php b/includes/class-Page.php index c32d069..d66ab55 100644 --- a/includes/class-Page.php +++ b/includes/class-Page.php @@ -1,228 +1,230 @@ . namespace itwikidelbot; use DateTime; use DateTimeZone; use cli\Input; use cli\Log; use wm\WikipediaIt; use mw\Tokens; /** * Handle a page */ class Page { /** * Time zone of Italian Wikipedia community. * * @var string */ const COMMUNITY_TIMEZONE = 'Europe/Rome'; /** * Enable this flag to ask for every changes * * @var bool */ public static $ASK_BEFORE_SAVING = false; /** * @var string Page title with prefix */ private $title; /** * Construct a Page * * @param $title Page title with its prefix */ public function __construct( $title ) { $this->title = $title; } /** * Get the page title with its prefix * * @return string */ public function getTitle() { return $this->title; } /** * Save this page * * @param $content string Page content * @param $summary string Edit summary * @return bool|mixed False if not created */ public function saveByContentSummary( $content, $summary ) { return static::saveByTitleContentSummary( $this->getTitle(), $content, $summary ); } /** * Save this page if it does not exist * * @param $content string Page content * @param $summary string Edit summary * @return bool|mixed False if not created */ public function saveByContentSummaryIfNotExists( $content, $summary ) { return self::saveByTitleContentSummaryIfNotExists( $this->getTitle(), $content, $summary ); } /** * Check if this page exists * * @return bool */ public function exists() { return static::existsByTitle( $this->getTitle() ); } /** * Save a page * * @param $title string Page name with prefix * @param $content string Page content * @param $summary string Edit summary * @return mixed Response */ public static function saveByTitleContentSummary( $title, $content, $summary ) { $api = self::api(); $args = [ 'action' => 'edit', 'title' => $title, 'text' => $content, 'summary' => $summary, 'token' => $api->login()->getToken( Tokens::CSRF ), 'bot' => 1, ]; if( self::$ASK_BEFORE_SAVING ) { print_r( $args ); if( 'y' !== Input::yesNoQuestion( "Save?" ) ) { $save = false; } } Log::info( "writing [[$title]]" ); return $api->post( $args ); } /** * Fetch the first revision date by direction * * @param $dir string direction * @return DateTime */ public function fetchFirstRevisionDateByDirection( $direction ) { $response = self::api()->fetch( [ 'action' => 'query', 'titles' => $this->getTitle(), 'prop' => 'revisions', 'rvprop' => [ 'timestamp' ], 'rvlimit' => 1, 'rvdir' => $direction, ] ); foreach( $response->query->pages as $page ) { - foreach( $page->revisions as $revision ) { - return self::createDateTimeFromString( $revision->timestamp ); + if( isset( $page->revisions ) ) { + foreach( $page->revisions as $revision ) { + return self::createDateTimeFromString( $revision->timestamp ); + } } } throw new \Exception( 'unable to fetch the creation date' ); } /** * Fetch the creation date of this page * * @return DateTime */ public function fetchCreationDate() { return $this->fetchFirstRevisionDateByDirection( 'newer' ); } /** * Fetch the creation date of this page * * @return DateTime */ public function fetchLasteditDate() { return $this->fetchFirstRevisionDateByDirection( 'older' ); } /** * Save a page if it does not exist * * @param $title string Page name with prefix * @param $content string Page content * @param $summary string Edit summary * @return bool|mixed False if not created */ public static function saveByTitleContentSummaryIfNotExists( $title, $content, $summary ) { if( ! static::existsByTitle( $title ) ) { return static::saveByTitleContentSummary( $title, $content, $summary ); } return false; } /** * Check if a page title exists * * @param $title string Page name with prefix * @return bool */ public static function existsByTitle( $title ) { $result = self::api()->fetch( [ 'action' => 'query', 'prop' => 'info', 'titles' => $title, ] ); foreach( $result->query->pages as $pageid => $page ) { return ! isset( $page->missing ); } return false; } /** * Get the API related to this page * * @return mw\API */ protected static function api() { return WikipediaIt::getInstance(); } /** * Create a DateTime object from a MediaWiki formatted date * * MediaWiki dates are formatted following the ISO8601 standard * and you may want to specify your community timezone. * * @param $datetime string * @return DateTime */ public static function createDateTimeFromString( $datetime ) { return DateTime::createFromFormat( DateTime::ISO8601, $datetime ) ->setTimezone( new DateTimeZone( self::COMMUNITY_TIMEZONE ) ); } } diff --git a/includes/class-PageYearMonthDayPDCs.php b/includes/class-PageYearMonthDayPDCs.php index f66d11e..4e7fbe6 100644 --- a/includes/class-PageYearMonthDayPDCs.php +++ b/includes/class-PageYearMonthDayPDCs.php @@ -1,101 +1,101 @@ . namespace itwikidelbot; /** * Abstraction of a page containing PDCs of a day */ abstract class PageYearMonthDayPDCs extends PageYearMonthDay { /** * PDCs indexed by type * * @var array */ private $pdcsByType; /** * Constructor * * @param $year int * @param $month int 1-12 * @param $day int 1-31 * @param $pdcs_by_type array PDCs by type * @see PageYearMonthDay::__construct() */ - public function __construct( $year, $month, $day, $pdcs_by_type ) { + public function __construct( $year, $month, $day, $pdcs_by_type = [] ) { $this->pdcsByType = $pdcs_by_type; parent::__construct( $year, $month, $day ); } /** * Static constructor * * @param $datetime DateTime * @param $pdcs_by_type PDCs by type * @return self */ - public static function createFromDateTimePDCs( $datetime, $pdcs_by_type ) { + public static function createFromDateTimePDCs( $datetime, $pdcs_by_type = [] ) { return new static( $datetime->format('Y'), $datetime->format('m'), $datetime->format('d'), $pdcs_by_type ); } /** * Get the PDCs indexed by type * * @return array */ public function getPDCsByType() { return $this->pdcsByType; } /** * Get the running PDCs indexed by type * * @return array */ public function getRunningPDCsByType() { return $this->getPDCsByTypeFilteringProtection( false ); } /** * Get the ended PDCs indexed by type * * @return array */ public function getEndedPDCsByType() { return $this->getPDCsByTypeFilteringProtection( true ); } /** * Get the PDCs indexed by type, but only which of them are protected (or not) * * @param $is_protected bool * @return array */ private function getPDCsByTypeFilteringProtection( $is_protected ) { $pdcs_by_type = $this->getPDCsByType(); foreach( $pdcs_by_type as $type => $pdcs ) { foreach( $pdcs as $i => $pdc ) { if( $pdc->isProtected() !== $is_protected ) { unset( $pdcs_by_type[ $type ][ $i ] ); } } } return $pdcs_by_type; } }