diff --git a/includes/class-Bot.php b/includes/class-Bot.php index a6f59b3..9df708e 100644 --- a/includes/class-Bot.php +++ b/includes/class-Bot.php @@ -1,293 +1,293 @@ . namespace itwikidelbot; use cli\Log; use DateTime; use DateInterval; use Exception; /** * The Bot class helps in running the bot */ class Bot { /** * The date * * @var DateTime */ private $dateTime; /** * A cache of already done dates. * It's an array of booleans like [year][month] = true. * * @var array */ private $cache = []; /** * Construct * * @param $date DateTime */ public function __construct( DateTime $date = null ) { if( ! $date ) { $date = new DateTime(); } $this->setDate( $date ); } /** * Static construct * * @param $date string */ public static function createFromString( $date = 'now' ) { return new self( new DateTime( $date ) ); } /** * Static construct * * @param $y int Year * @param $m int Month 1-12 * @param $d int Day 1-31 */ public static function createFromYearMonthDay( $y, $m, $d ) { return new self( DateTime::createFromFormat( "Y m d", "$y $m $d") ); } /** * Get the date * * @return DateTime */ public function getDate() { return $this->dateTime; } /** * Set the date * * @param $date DateTime * @return self */ public function setDate( DateTime $date ) { $this->dateTime = $date; } /** * Add a day * * @return self */ public function nextDay() { return $this->addDays( 1 ); } /** * Add a day * * @return self */ public function previousDay() { return $this->subDays( 1 ); } /** * Add a certain number of days * * @param $days int * @return self */ public function addDays( $days ) { $this->getDate()->add( new DateInterval( sprintf( 'P%dD', $days ) ) ); return $this; } /** * Subtract a certain number of days * * @param $days int * @return self */ public function subDays( $days ) { $this->getDate()->sub( new DateInterval( sprintf( 'P%dD', $days ) ) ); return $this; } /** * Fetch the last bot edit on the next page * * @return DateTime|null */ public function fetchLastedit() { $lastedit = null; try { $lastedit = PageYearMonthDayPDCsCount::createFromDateTime( $this->getDate() ) ->fetchLasteditDate(); } catch( PDCMissingException $e ) { // Unexisting. OK. Log::debug( $e->getMessage() ); } catch( PDCWithoutCreationDateException $e ) { // Happened once. Just try to create. Log::debug( $e->getMessage() ); } return $lastedit; } /** * Is the last edit date older than some seconds? (on the next page) * * @param $seconds int * @return bool */ public function isLasteditOlderThanSeconds( $seconds ) { $lastedit = $this->fetchLastedit(); if( ! $lastedit ) { return true; // Unexisting. OK. } return time() - $lastedit->format( 'U' ) > $seconds; } /** * Is the last edit date older than some minutes? (on the next page) * * @param $minutes int * @return bool */ public function isLasteditOlderThanMinutes( $minutes ) { return $this->isLasteditOlderThanSeconds( 60 * $minutes ); } /** * Run the bot at the internal date * * @TODO: do not repeat twice the same yearly and montly categories * @return self */ public function run() { $cache = & $this->cache; // date initialization $date = $this->getDate(); $year = $date->format( 'Y' ); $month = $date->format( 'n' ); // 1-12 $day = $date->format( 'j' ); // yearly category $y_category = new CategoryYear( $year ); // monthly category $m_category = new CategoryYearMonth( $year, $month ); // create all the PDC types $category_types = []; foreach( CategoryYearMonthDayTypes::all() as $CategoryType ) { $category_types[] = new $CategoryType( $year, $month, $day ); } // all the categories $all_categories = $category_types; if( ! isset( $cache[ $month ] ) ) { $all_categories[] = $m_category; } if( ! isset( $cache[ $year ] ) ) { $all_categories[] = $y_category; } // check in bulk if the categories already exist Pages::populateWheneverTheyExist( $all_categories ); // create the yearly category (once) if( ! isset( $cache[ $year ] ) ) { $cache[ $year ] = []; $y_category->saveIfNotExists(); } // create the monthly category (once) $cache = & $cache[ $year ]; if( ! isset( $cache[ $month ] ) ) { $cache[ $month ] = true; $m_category->saveIfNotExists(); } Log::info( "work on $year/$month/$day" ); // PDCs indexed by page ID $pdcs = []; // handle every PDC type foreach( $category_types as $category_type ) { // fetch PDCs from this type $category_type_pdcs = $category_type->fetchPDCs(); // save the specific daily category type only if it's not empty // ...or only if it's the main category (that can be without pages) - if( $category_type_pdcs || get_class( $category_type ) === CategoryYearMonthDay::class ) { + if( $category_type_pdcs || $category_type->getShouldBeCreatedEvenIfEmpty() ) { $category_type->saveIfNotExists(); } // merge the same PDCs into one foreach( $category_type_pdcs as $category_type_pdc ) { $id = $category_type_pdc->getID(); if( isset( $pdcs[ $id ] ) ) { $pdcs[ $id ]->merge( $category_type_pdc ); } else { $pdcs[ $id ] = $category_type_pdc; } } } // select only the PDCs that belong to this date $pdcs = PDCs::filterByDate( $pdcs, $date ); Log::info( sprintf( sprintf( "found %d PDCs", count( $pdcs ) ) ) ); // populate missing informations PDCs::populateMissingInformations( $pdcs ); // sort by creation date PDCs::sortByCreationDate( $pdcs ); // index then by their PDC_TYPE $pdcs_by_type = PDCs::indexByType( $pdcs ); // save the counting page PageYearMonthDayPDCsCount::createFromDateTimePDCs( $this->getDate(), $pdcs_by_type ) ->save(); // save the log page PageYearMonthDayPDCsLog::createFromDateTimePDCs( $this->getDate(), $pdcs_by_type ) ->save(); return $this; } } diff --git a/includes/class-CategoryYearMonthDay.php b/includes/class-CategoryYearMonthDay.php index 6dfb856..f1beabd 100644 --- a/includes/class-CategoryYearMonthDay.php +++ b/includes/class-CategoryYearMonthDay.php @@ -1,293 +1,301 @@ . namespace itwikidelbot; use cli\Log; /** * Handle a daily category that directly contains semplified ("semplificate") PDC pages * It contains other sub-categories. * * e.g. https://it.wikipedia.org/wiki/Categoria:Cancellazioni_del_19_febbraio_2018 */ class CategoryYearMonthDay extends PageYearMonthDay { /** * Template name * * @override CategoryTemplated::TEMPLATE_NAME */ const TEMPLATE_NAME = 'CATEGORY_DAY'; /** * PDC type * * The part of the category title that rappresent this type of PDC. */ const PDC_TYPE = 'semplificate'; /** * PDC type (in an human form) * * Abbreviation of the PDC_TYPE. It's used in the counting page. * * @var string */ const PDC_TYPE_HUMAN = 'semplificata'; /** * Title format * * Used to describe both the plain text title and its matching pattern. * * Don't use placeholders different from '%s'. * * Arguments: * 1: day * 2: month name * 3: year */ const TITLE_FORMAT = 'Categoria:Cancellazioni del %s %s %s'; /** * Title format arguments * * Arguments that can create this page title, when filling the title format. * * @return array Arguments for the TITLE_FORMAT */ protected function getTitleFormatArguments() { return [ $this->getDay(), // 1: day $this->getMonthName(), // 2: month name $this->getYear(), // 3: year ]; } /** * Title format pattern groups * * Regex groups that can create a regex for this page title, when filling the title format. * * @return array Array of regex groups for the TITLE_FORMAT */ protected static function titleFormatGroups() { return [ '([0-9]{1,2})', // 1: day '([a-z]+)', // 2: month name '([0-9]{4})', // 3: year ]; } /** * Title regex * * Complete regex that can match arguments (day, month, etc.) from a page title. * * @return string regex */ protected static function titleRegex() { return '/^' . vsprintf( static::TITLE_FORMAT, static::titleFormatGroups() ) . '$/'; } /** * Get the title of this page * * It's obtained filling the title format with its arguments. * * @override PageTemplated#getTemplatedTitle() * @return string */ public function getTemplatedTitle() { return vsprintf( static::TITLE_FORMAT, $this->getTitleFormatArguments() ); } /** * Static constructor * * Create an CategoryYearMonthDay object extracting informations from a specific page title (if it matches the title format). * * @param $title string Page title to be matched * @return self|false */ public static function createParsingTitle( $title ) { if( 1 === preg_match( static::titleRegex(), $title, $matches ) ) { // discard the first match group: it's simply the $title itself array_shift( $matches ); return static::createFromTitleFormatArguments( $matches ); } return false; } /** * Static constructor * * Create a CategoryYearMonthDay object from its title format arguments * * @see self::createFromTitle() * @see PageYearMonth::__construct() * @param $arguments Arguments for the page title format * @return self */ protected static function createFromTitleFormatArguments( $arguments ) { list( $day, $month_name, $year ) = $arguments; return new static( (int) $year, Months::name2number( $month_name ) + 1, (int) $day ); } /** * Template arguments: * * 1: category title * 2: year * 3: month 1-12 * 4: month name * 5: day 1-31 * * @override CategoryYearMonth::getTemplateArguments() */ public function getTemplateArguments() { return array_merge( [ ( new CategoryYearMonth( $this->getYear(), $this->getMonth() ) ) ->getTemplatedTitle() ], parent::getTemplateArguments() ); } /** * Fetch PDCs from this category * * @return array */ public function fetchPDCs() { $api = self::api()->createQuery( [ 'action' => 'query', // generator=categorymembers: get pages in category // gmtitle=: specify the category title // gmtype=page: get sub-pages // gmsort=timestamp: order by insertion date in that category // gmdir=asc: ascending order // https://it.wikipedia.org/w/api.php?action=help&modules=query%2Bcategorymembers // Note: Generator parameter names must be prefixed with a 'g' 'generator' => 'categorymembers', 'gcmtitle' => $this->getTitle(), // 'gcmtype' => 'page', // Note: Ignored when cmsort=timestamp is set. 'gcmnamespace' => 4, // Wikipedia 'gcmlimit' => 100, 'gcmsort' => 'timestamp', 'gcmdir' => 'asc', // // for each page load infos, categories and latest revision // https://it.wikipedia.org/w/api.php?action=help&modules=query%2Binfo // https://it.wikipedia.org/w/api.php?action=help&modules=query%2Bcategories // https://it.wikipedia.org/w/api.php?action=help&modules=query%2Brevisions // Note: probably the revisions parameter is unuseful for last update date (already provided by "touched"). // Note: revisions can be still useful to know the creation date 'prop' => [ 'info' , 'categories'/*, 'revisions'*/ ], // inprop=protecion: list of the protection level of each page // https://it.wikipedia.org/w/api.php?action=help&modules=query%2Bcategories 'inprop' => 'protection', // clprop=sortkey: adds the sortkey and sortkey prefix for the category // clprop=timestamp: adds the timestamp of when the page was included // https://it.wikipedia.org/w/api.php?action=help&modules=query%2Binfo 'clprop' => [ 'sortkey', 'timestamp' ], // TODO: remove this workaround after the core API has been fixed. // https://phabricator.wikimedia.org/T433922#12184368 'cllimit' => 500, // rvprop=timestamp: timestamp of the revision // rvlimit=1: only 1 revision // rvdir=older: order by oldest // https://it.wikipedia.org/w/api.php?action=help&modules=query%2Brevisions // 'rvprop' => 'timestamp', // 'rvdir' => 'older', // 'rvlimit' => 1, ] ); $all = []; while( $api->hasNext() ) { $next = $api->fetchNext(); if( isset( $next->query->pages ) ) { foreach( $next->query->pages as $page ) { // multiple call will add more infos on the same page // https://www.mediawiki.org/wiki/API:Query#Generators_and_continuation // TODO: use batchcomplete for better memory usage $pageid = $page->pageid; if( isset( $all[ $pageid ] ) ) { foreach( $page as $property => $value ) { if( isset( $all[ $pageid ]->{$property} ) && is_array( $all[ $pageid ]->{$property} ) ) { // merge categories $all[ $pageid ]->{$property} = array_merge( $all[ $pageid ]->{$property}, $value ); } else { $all[ $pageid ]->{$property} = $value; } } } else { $all[ $pageid ] = $page; } } } } $pdcs = []; foreach( $all as $page ) { try { $pdcs[] = PDC::createFromRaw( $page ); } catch( PDCException $e ) { Log::warn( sprintf( "exception in PDC '%s': %s", $page->title, $e->getMessage() ) ); } } return $pdcs; } + public function getShouldBeCreatedEvenIfEmpty(): bool + { + // The generic category 'Categoria:Cancellazioni del %s %s %s' + // should be created even if empty. + // Please override this method as child categories may have different opinions. + return true; + } + /** * Get a "genericity" score of this PDC category type. * * @see CategoryYearMonthDayTypes::genericityFromClass() * @return int */ public static function genericity() { return CategoryYearMonthDayTypes::genericityFromClass( static::class ); } } diff --git a/includes/class-CategoryYearMonthDayType.php b/includes/class-CategoryYearMonthDayType.php index fb9fb6f..2102906 100644 --- a/includes/class-CategoryYearMonthDayType.php +++ b/includes/class-CategoryYearMonthDayType.php @@ -1,107 +1,114 @@ . namespace itwikidelbot; /** * Abstraction of a daily category with a specified PDC type */ abstract class CategoryYearMonthDayType extends CategoryYearMonthDay { /** * Template name of this category * * @override CategoryTemplated::TEMPLATE_NAME */ const TEMPLATE_NAME = 'CATEGORY_DAY_PDCTYPE'; /** * Title format * * Used to describe both the plain text title and its matching pattern. * * Don't use placeholders different from '%s'. * * Arguments: * 1: PDC type * 4: day * 3: human month * 2: year * * @override CategoryYearMonthDay::TITLE_FORMAT */ const TITLE_FORMAT = 'Categoria:Cancellazioni %s del %s %s %s'; /** * Title format arguments * * Arguments that can create this page title, when filling the title format. * * @return array Arguments for the TITLE_FORMAT * @override CategoryYearMonthDay::getTitleFormatArguments() */ protected function getTitleFormatArguments() { return [ static::PDC_TYPE, // 1: PDC type $this->getDay(), // 2: day $this->getMonthName(), // 3: month name $this->getYear(), // 4: year ]; } /** * Title format regex groups * * Regex groups that can create a regex for this page title, when filling the title format. * * @return array Array of regex groups for the TITLE_FORMAT * @override CategoryYearMonthDay::titleFormatGroups() */ protected static function titleFormatGroups() { return [ preg_quote( static::PDC_TYPE ), // 1: PDC type (it's correct that is not grouped) '([0-9]{1,2})', // 2: day '([a-z]+)', // 3: month name '([0-9]{4})', // 4: year ]; } /** * Get template arguments * * 1: category title * 2: year * 3: month 1-12 * 4: month name * 5: day 1-31 * 6: PDC type * * @override CategoryTemplated::getTemplateArguments() */ public function getTemplateArguments() { $parent_arguments = parent::getTemplateArguments(); $parent_arguments[] = static::PDC_TYPE; return $parent_arguments; } + #[Override] + public function getShouldBeCreatedEvenIfEmpty(): bool + { + // All specific categories like 'Categoria:Cancellazioni %s del %s %s %s' + // should not be created if are empty. + return false; + } }