diff --git a/includes/class-CategoryYearMonthDay.php b/includes/class-CategoryYearMonthDay.php index 4c29c22..acd037a 100644 --- a/includes/class-CategoryYearMonthDay.php +++ b/includes/class-CategoryYearMonthDay.php @@ -1,54 +1,69 @@ . namespace itwikidelbot; /** * Handle a daily category that directly contains semplified ("semplificate") PDC pages * It contains other sub-categories. * * e.g. https://it.wikipedia.org/wiki/Categoria:Cancellazioni_del_19_febbraio_2018 */ class CategoryYearMonthDay extends PageYearMonthDay { /** * Template name * * @override CategoryTemplated::TEMPLATE_NAME */ const TEMPLATE_NAME = 'CATEGORY_DAY'; /** - * Template arguments + * Template arguments: + * + * 1: category title + * 2: year + * 3: month 1-12 + * 4: month name + * 5: day 1-31 * * @override CategoryYearMonth::getTemplateArguments() */ public function getTemplateArguments() { return array_merge( [ ( new CategoryYearMonth( $this->getYear(), $this->getMonth() ) ) ->getTemplatedTitle() ], parent::getTemplateArguments() ); } + /** + * Does a certain specific category name match the pattern of this category? + * + * @param $category_name string + * @return bool + */ + public static function matchCategoryName( $category_name ) { + return 1 === preg_match( '/^Categoria:Cancellazioni del [0-9]+ [a-z]+ [0-9]+$/', $category_name ); + } } diff --git a/includes/class-CategoryYearMonthDayType.php b/includes/class-CategoryYearMonthDayType.php index 0aa7a29..91b51df 100644 --- a/includes/class-CategoryYearMonthDayType.php +++ b/includes/class-CategoryYearMonthDayType.php @@ -1,171 +1,165 @@ . namespace itwikidelbot; use \cli\Log; /** * Abstraction of a daily category with a specified PDC type */ abstract class CategoryYearMonthDayType extends CategoryYearMonthDay { /** * Template name of this category * * @override CategoryTemplated::TEMPLATE_NAME */ const TEMPLATE_NAME = 'CATEGORY_DAY_PDCTYPE'; /** * PDC type e.g. 'consensuale' * * To be overrided */ const PDC_TYPE = 'EXAMPLE'; /** * Get the PDC type * * @return string e.g. 'consensuale' */ public function getPDCType() { return static::PDC_TYPE; } /** * Get template arguments * * @override CategoryTemplated::getTemplateArguments() */ public function getTemplateArguments() { $parent_arguments = parent::getTemplateArguments(); $parent_arguments[] = $this->getPDCType(); return $parent_arguments; } /** * Create a PDF of this type (from raw) * * @param $pdc_raw mixed Raw PDC data * @see PDC::createFromRaw() */ public function createPDCFromRaw( $pdc_raw ) { return PDC::createFromRaw( $this, $pdc_raw ); } /** * Fetch PDCs of this type * * @return array */ public function fetchPDCs() { $api = self::api()->createQuery( [ 'action' => 'query', // generator=categorymembers: get pages in category // gmtitle=: specify the category title // gmtype=page: get sub-pages // gmsort=timestamp: order by insertion date in that category // gmdir=asc: ascending order // https://it.wikipedia.org/w/api.php?action=help&modules=query%2Bcategorymembers // Note: Generator parameter names must be prefixed with a 'g' 'generator' => 'categorymembers', 'gcmtitle' => $this->getTitle(), - 'gcmtype' => 'page', // Note: Ignored when cmsort=timestamp is set. +// 'gcmtype' => 'page', // Note: Ignored when cmsort=timestamp is set. + 'gcmnamespace' => 4, // Wikipedia 'gcmlimit' => 100, 'gcmsort' => 'timestamp', 'gcmdir' => 'asc', // // for each page load infos, categories and latest revision // https://it.wikipedia.org/w/api.php?action=help&modules=query%2Binfo // https://it.wikipedia.org/w/api.php?action=help&modules=query%2Bcategories // https://it.wikipedia.org/w/api.php?action=help&modules=query%2Brevisions // Note: probably the revisions parameter is unuseful for last update date (already provided by "touched"). // Note: revisions can be still useful to know the creation date 'prop' => [ 'info' , 'categories'/*, 'revisions'*/ ], // inprop=protecion: list of the protection level of each page // https://it.wikipedia.org/w/api.php?action=help&modules=query%2Bcategories 'inprop' => 'protection', // clprop=sortkey: adds the sortkey and sortkey prefix for the category // clprop=timestamp: adds the timestamp of when the page was included // https://it.wikipedia.org/w/api.php?action=help&modules=query%2Binfo 'clprop' => [ 'sortkey', 'timestamp' ], // rvprop=timestamp: timestamp of the revision // rvlimit=1: only 1 revision // rvdir=older: order by oldest // https://it.wikipedia.org/w/api.php?action=help&modules=query%2Brevisions // 'rvprop' => 'timestamp', // 'rvdir' => 'older', // 'rvlimit' => 1, ] ); $all = []; while( $api->hasNext() ) { $next = $api->fetchNext(); if( isset( $next->query->pages ) ) { foreach( $next->query->pages as $page ) { // multiple call will add more infos on the same page // https://www.mediawiki.org/wiki/API:Query#Generators_and_continuation // TODO: use batchcomplete for better memory usage - // Note: pages are not directly saved with their pageid to preserve their order $pageid = $page->pageid; if( isset( $all[ $pageid ] ) ) { foreach( $page as $property => $value ) { $all[ $pageid ]->{$property} = $value; } } else { $all[ $pageid ] = $page; } } } } $pdcs = []; foreach( $all as $page ) { try { $pdcs[] = $this->createPDCFromRaw( $page ); - } catch( PDCExpiredException $e ) { - Log::info( sprintf( - "%s: %s", - $page->title, - $e->getMessage() - ) ); } catch( PDCException $e ) { Log::warn( sprintf( "exception in PDC '%s': %s", $page->title, $e->getMessage() ) ); } } return $pdcs; } } diff --git a/includes/class-CategoryYearMonthDayTypeConsensual.php b/includes/class-CategoryYearMonthDayTypeConsensual.php index 71b531c..7e1d251 100644 --- a/includes/class-CategoryYearMonthDayTypeConsensual.php +++ b/includes/class-CategoryYearMonthDayTypeConsensual.php @@ -1,39 +1,39 @@ . namespace itwikidelbot; /** * Handle a daily consensual PDC category * * e.g. https://it.wikipedia.org/wiki/Categoria:Cancellazioni_consensuali_del_13_aprile_2018 */ class CategoryYearMonthDayTypeConsensual extends CategoryYearMonthDayType { /** * PDC type * * @override CategoryYearMonthDayType::PDC_TYPE */ - const PDC_TYPE = 'consensuale'; + const PDC_TYPE = 'consensuali'; } diff --git a/includes/class-CategoryYearMonthDayTypes.php b/includes/class-CategoryYearMonthDayTypes.php index 8762fa5..d0727f5 100644 --- a/includes/class-CategoryYearMonthDayTypes.php +++ b/includes/class-CategoryYearMonthDayTypes.php @@ -1,112 +1,122 @@ . namespace itwikidelbot; +use \cli\Log; + /** * Handler of all the PDCs in this day of the year */ class CategoryYearMonthDayTypes extends CategoryYearMonthDay { /** * PDC types * * Categories in order of importance (I think) * * @var array */ public static $TYPES = [ CategoryYearMonthDayTypeVoting ::class, CategoryYearMonthDayTypeProlonged ::class, CategoryYearMonthDayTypeConsensual::class, CategoryYearMonthDayTypeOrdinary ::class, CategoryYearMonthDayTypeSimple ::class, ]; /** - * Return the more precise PDC type between two + * Return the newer PDC, but setting the older start date. * - * @param $type1 Class name - * @param $type2 Class name - * @return Class name + * @param $pdc1 PDC + * @param $pdc2 PDC + * @return $pdc PDC */ - public static function bestType( $type1, $type2 ) { - $i1 = array_search( self::$TYPES, $type1 ); - $i2 = array_search( self::$TYPES, $type2 ); - if( $i1 === false || $i2 === false ) { - throw new \InvalidArgumentException( 'cannot compare unknown types' ); + public function mergePDCs( $pdc1, $pdc2 ) { + if( $pdc2->getDurationDays() > $pdc1->getDurationDays() ) { + return $pdc2->setStartDate( $pdc1->getStartDate() ); } - return $i1 < $i2 ? $type1 : $type2; + return $pdc1; } /** * Operate on every PDC type */ public function run() { $pdcs_by_id = []; foreach( self::$TYPES as $Category ) { $category = new $Category( $this->getYear(), $this->getMonth(), $this->getDay() ); $pdcs = $category->fetchPDCs(); if( count( $pdcs ) ) { $category->saveIfNotExists(); } // merge the same PDCs foreach( $pdcs as $pdc ) { $id = $pdc->getId(); + if( $id == '7156617' ) { + var_dump( $category, $pdc ); + } if( isset( $pdcs_by_id[ $id ] ) ) { - $pdcs_by_id[ $id ]->setType( self::bestType( - $pdcs_by_id[ $id ]->getTypeClass(), - $pdc ->getTypeClass() - ) ); + $pdcs_by_id[ $id ] = self::mergePDCs( $pdcs_by_id[ $id ], $pdc ); } else { $pdcs_by_id[ $id ] = $pdc; } } } // sort PDCs by start date usort( $pdcs_by_id, function ( $a, $b ) { - return $a->getStartDate() < $b->getStartDate(); + return $a->getStartDate() > $b->getStartDate(); } ); + foreach( $pdcs_by_id as $i => $pdc ) { + if( $pdc->getDurationDays() > 7 ) { + Log::info( sprintf( + 'the PDC %s is expired after %d days', + $pdc->getTitle(), + $pdc->getDurationDays() + ) ); + } + } + // index PDcs by their type $pdcs_by_type = []; foreach( $pdcs_by_id as $pdc ) { $type = $pdc->getType(); if( ! isset( $pdcs_by_type[ $type ] ) ) { $pdcs_by_type[ $type ] = []; } $pdcs_by_type[ $type ][] = $pdc; } PageYearMonthDayPDCsCount::createFromDateTimePDCs( $this->getDateTime(), $pdcs_by_type ) ->save(); PageYearMonthDayPDCsLog::createFromDateTimePDCs( $this->getDateTime(), $pdcs_by_type ) ->save(); } } diff --git a/includes/class-PDC.php b/includes/class-PDC.php index 70ec1f2..fba3e91 100644 --- a/includes/class-PDC.php +++ b/includes/class-PDC.php @@ -1,422 +1,438 @@ . namespace itwikidelbot; use DateTime; /** * Handle a "Procedura di cancellazione" * * It is a page like "Wikipedia:Pagine da cancellare/ASD" */ class PDC extends Page { /** * Prefix of every PDC */ const PREFIX = 'Wikipedia:Pagine da cancellare/'; /** * Prefix of every multiple PDC */ const PREFIX_MULTIPLE = self::PREFIX . 'multiple/'; /** * PDC category type * * @var CategoryYearMonthDayType */ private $categoryType; /** * Page id * * @var int */ private $id; /** * Title of the PDC subject page (the Wikipedia article title) * * Obtained from {{DEFAULTSORT}} category sortkey * * @var string */ private $titleSubject; /** * Page length in bytes * * @var int */ private $length; /** * When the PDC was added to the PDC type category * * @var date */ private $startDate; /** * Last update * * @var date */ private $lasteditDate; /** * Is Protected? * * @var bool */ private $isProtected; /** * Properties that the raw object must have * * @var array */ private static $RAW_PROPERTIES = [ 'pageid', 'title', 'length', 'touched', 'protection', 'categories' ]; /** * Constructor * * @param $category_type CategoryYearMonthDayType PDC category type * @param $id int PDC page id * @param $title string PDC title prefixed * @param $title_subject string Title of the subject page (the Wikipedia article title) * @param $length int PDC length * @param $start DateTime When the PDC was added to the PDC type category * @param $lastedit DateTime PDC lastedit date * @param $is_protected bool Is the PDC protected? * @see Page::__construct() * @throws PDCException */ public function __construct( $category_type, $id, $title, $title_subject, $length, DateTime $start, DateTime $lastedit, $is_protected ) { $this->categoryType = $category_type; $this->id = $id; $this->titleSubject = $title_subject; $this->length = $length; - $this->startDate = $start; $this->lasteditDate = $lastedit; $this->isProtected = $is_protected; + $this->setStartDate( $start ); parent::__construct( $title ); // consistence checks if( ! $this->isTitlePrefixValid() ) { throw new PDCException( 'not a PDC' ); } - if( $this->getDurationDays() > 7 ) { - throw new PDCExceptionExpired( 'not anymore a PDC of type ' . $this->getType() ); - } + + //if( $this->getDurationDays() > 7 ) { + // throw new PDCExceptionExpired( 'not anymore a PDC of type ' . $this->getType() ); + //} $this->checkTitleSubjectConsistence(); } /** * Get the page id * * @return int */ public function getId() { return $this->id; } /** * Get the title of the subject page (the Wikipedia article title) * * Obtained from {{DEFAULTSORT}} category sortkey * * @return string */ public function getTitleSubject() { return $this->titleSubject; } /** * Get the page length * * @return int */ public function getLength() { return $this->length; } /** * Get the start date of this PDC (of this type) * * @return DateTime */ public function getStartDate() { return $this->startDate; } + /** + * set the start date of this PDC (of this type) + * + * @param $start_date DateTime + * @return self + */ + public function setStartDate( DateTime $start_date ) { + $this->startDate = $start_date; + return $this; + } + /** * Get the latest update date * * @return DateTime */ public function getLasteditDate() { return $this->lasteditDate; } /** * Get the PDC type class * * @return class */ public function getTypeClass() { return $this->categoryType; } /** * Get the PDC type name * * @return string e.g. 'consensuale' */ public function getType() { return $this->getTypeClass()::getPDCType(); } /** * Check if this page is protected * * @return bool */ public function isProtected() { return $this->isProtected; } /** * Check if this PDC has a known prefix * * @return bool */ private function isTitlePrefixValid() { return $this->titlehasPrefix( self::PREFIX ); } /** * Check if this PDC involves multiple PDCs * * @return bool */ public function isMultiple() { return $this->titleHasPrefix( self::PREFIX_MULTIPLE ); } /** * Get the title prefix * * @return string */ private function getTitlePrefix() { return $this->isMultiple() ? self::PREFIX_MULTIPLE : self::PREFIX; } /** * Get the title unprefixed * * @return string */ private function getTitleUnprefixed() { return substr( $this->getTitle(), strlen( $this->getTitlePrefix() ) ); } /** * Check if the title has a certain prefix * * @param $prefix string * @return bool */ private function titleHasPrefix( $prefix ) { return 0 === strpos( $this->getTitle(), $prefix ); } /** * Verify if the title of the subject is consistent with the PDC title * * @return bool */ private function checkTitleSubjectConsistence() { // subject from {{DEFAULTSORT}} category prefix $subject = $this->getTitleSubject(); // subject generated unprefixing page title $subject_gen = $this->getTitleUnprefixed(); if( $subject !== $subject_gen ) { // Check if the PDC title ends with an enumeration e.g. 'asd/14' $status = preg_match( '@(.*)/[0-9]+$@', $subject_gen, $matches ); if( 1 === $status ) { // Get the interesting part e.g. 'asd' $subject_gen = $matches[ 1 ]; } } if( $subject !== $subject_gen ) { throw new PDCException( sprintf( 'the sortkey is "%s" and it is inconsistent with the PDC suffix "%s"', $subject_gen, $subject ) ); } } /** * Static constructor * * @param $type CategoryYearMonthDayType Specified PDC type * @param $page mixed * @return self * @throws PDCException */ public static function createFromRaw( $type, $page ) { // verify consistence foreach( self::$RAW_PROPERTIES as $property ) { if( ! isset( $page->$property ) ) { throw new PDCException( "PDC has not property $property" ); } } // read the title of the subject from the {{DEFAULTSORT}} category sortkey prefix + $title_subject = null; + foreach( $page->categories as $category ) { + $title_subject = $category->sortkeyprefix; + break; + } + if( ! isset( $title_subject ) ) { + throw new PDCException( 'missing category sortkeyprefix that contains title subject' ); + } + // get the start date of this PDC from the timestamp - $title_subject = null;; $date_added_in_category = null; foreach( $page->categories as $category ) { - if( ! isset( $category->title ) ) { - throw new PDCException( 'missing category title' ); - } - if( $category->title === $type->getTitle() ) { - $title_subject = $category->sortkeyprefix; + if( CategoryYearMonthDay::matchCategoryName( $category->title ) ) { + if( ! isset( $category->timestamp ) ) { + throw new PDCException( 'missing category timestamp' ); + } $date_added_in_category = $category->timestamp; + break; } - break; - } - if( ! isset( $title_subject ) ) { - throw new PDCException( 'missing category sortkey prefix that contains title subject' ); } if( ! isset( $date_added_in_category ) ) { - throw new PDCException( 'missing timestamp that contains the PDC start date' ); + throw new PDCException( 'missing timestamp of insertion into the daily category' ); } // read protection status $is_protected = false; foreach( $page->protection as $protection ) { if( 'edit' === $protection->type && 'sysop' === $protection->level ) { $is_protected = true; break; } } return new self( $type, (int) $page->pageid, $page->title, $title_subject, (int) $page->length, DateTime::createFromFormat( DateTime::ISO8601, $date_added_in_category ), DateTime::createFromFormat( DateTime::ISO8601, $page->touched ), $is_protected ); } /** * Get the PDC temperature * * The temperature is a value betweeen 0-100 * * @return int */ public function getTemperature() { $slope = null; $offset = null; switch( $this->getType() ) { case CategoryYearMonthDayTypeSimple::getPDCType(): $slope = 0.0365; $offset = -24.0; break; case CategoryYearMonthDayTypeConsensual::getPDCType(): $slope = 0.0075; $offset = -12.81; break; case CategoryYearMonthDayTypeProlonged::getPDCType(): $slope = 0.0035; $offset = -9.76; break; case CategoryYearMonthDayTypeOrdinary::getPDCType(): case CategoryYearMonthDayTypeVoting ::getPDCType(): $slope = 0.0025; $offset = -16.43; break; default: throw new PDCException( 'unexpected type' ); } $temp = round( $slope * $this->getLength() + $offset ); if ( $temp > 100 ) { $temp = 100; } elseif ( $temp < 0 ) { $temp = 0; } return $temp; } /** * Get the number of days of duration of this PDC * * @return int */ public function getDurationDays() { - return (int) $this->getStartDate()->diff( $this->getLasteditDate() )->format('%a'); + return (int) $this->getLasteditDate()->diff( $this->getStartDate() )->format('%a'); } /** * Get an human rappresentation of the duration of this PDC * * @return string */ public function getHumanDuration() { $days = $this->getDurationDays(); if( 0 === $days ) { return Template::get( 'DURATION.hours' ); } if( 1 === $days ) { return Template::get( 'DURATION.day' ); } return Template::get( 'DURATION.days', [ $days ] ); } }