diff --git a/includes/class-Bot.php b/includes/class-Bot.php index b91dee9..ed1a5f3 100644 --- a/includes/class-Bot.php +++ b/includes/class-Bot.php @@ -1,266 +1,266 @@ . namespace itwikidelbot; use cli\Log; use DateTime; use DateInterval; use Exception; /** * The Bot class helps in running the bot */ class Bot { /** * The date * * @var DateTime */ private $dateTime; /** * A cache of already done dates. * It's an array of booleans like [year][month] = true. * * @var array */ private $cache = []; /** * Construct * * @param $date DateTime */ public function __construct( DateTime $date = null ) { if( ! $date ) { $date = new DateTime(); } $this->setDate( $date ); } /** * Static construct * * @param $date string */ public static function createFromString( $date = 'now' ) { return new self( new DateTime( $date ) ); } /** * Static construct * * @param $y int Year * @param $m int Month 1-12 * @param $d int Day 1-31 */ public static function createFromYearMonthDay( $y, $m, $d ) { return new self( DateTime::createFromFormat( "Y m d", "$y $m $d") ); } /** * Get the date * * @return DateTime */ public function getDate() { return $this->dateTime; } /** * Set the date * * @param $date DateTime * @return self */ public function setDate( DateTime $date ) { $this->dateTime = $date; } /** * Add a day * * @return self */ public function nextDay() { return $this->addDays( 1 ); } /** * Add a day * * @return self */ public function previousDay() { return $this->subDays( 1 ); } /** * Add a certain number of days * * @param $days int * @return self */ public function addDays( $days ) { $this->getDate()->add( new DateInterval( sprintf( 'P%dD', $days ) ) ); return $this; } /** * Subtract a certain number of days * * @param $days int * @return self */ public function subDays( $days ) { $this->getDate()->sub( new DateInterval( sprintf( 'P%dD', $days ) ) ); return $this; } /** * Fetch the last bot edit on the next page * * @return DateTime|null */ public function fetchLastedit() { $lastedit = null; try { $lastedit = PageYearMonthDayPDCsCount::createFromDateTime( $this->getDate() ) ->fetchLasteditDate(); } catch( Exception $e ) { // Unexisting. OK. } return $lastedit; } /** * Is the last edit date older than some seconds? (on the next page) * * @param $seconds int * @return bool */ public function isLasteditOlderThanSeconds( $seconds ) { $lastedit = $this->fetchLastedit(); if( ! $lastedit ) { return true; // Unexisting. OK. } return time() - $lastedit->format( 'U' ) > $seconds; } /** * Is the last edit date older than some minutes? (on the next page) * * @param $minutes int * @return bool */ public function isLasteditOlderThanMinutes( $minutes ) { return $this->isLasteditOlderThanSeconds( 60 * $minutes ); } /** * Run the bot at the internal date * * @TODO: do not repeat twice the same yearly and montly categories * @return self */ public function run() { // date initialization $date = $this->getDate(); $year = $date->format( 'Y' ); $month = $date->format( 'n' ); // 1-12 $day = $date->format( 'j' ); // create the yearly category (once) $cache = & $this->cache; if( ! isset( $cache[ $year ] ) ) { $cache[ $year ] = []; ( new CategoryYear( $year ) ) ->saveIfNotExists(); } // create the monthly category (once) $cache = & $cache[ $year ]; if( ! isset( $cache[ $month ] ) ) { $cache[ $month ] = true; ( new CategoryYearMonth( $year, $month ) ) ->saveIfNotExists(); } Log::info( "work on $year/$month/$day" ); // PDCs indexed by page ID $pdcs = []; // handle every PDC type foreach( CategoryYearMonthDayTypes::all() as $CategoryType ) { // fetch PDCs from this type $category_type = new $CategoryType( $year, $month, $day ); $category_type_pdcs = $category_type->fetchPDCs(); // save the specific daily category type only if it's not empty // ...or only if it's the main category (that can be without pages) if( $category_type_pdcs || get_class( $category_type ) === CategoryYearMonthDay::class ) { $category_type->saveIfNotExists(); } // merge the same PDCs into one foreach( $category_type_pdcs as $category_type_pdc ) { $id = $category_type_pdc->getID(); if( isset( $pdcs[ $id ] ) ) { $pdcs[ $id ]->merge( $category_type_pdc ); } else { $pdcs[ $id ] = $category_type_pdc; } } } // select only the PDCs that belong to this date $pdcs = PDCs::filterByDate( $pdcs, $date ); Log::info( sprintf( sprintf( "found %d PDCs", count( $pdcs ) ) ) ); - // populate the 'subject themes' PDC field - PDCs::populateSubjectThemes( $pdcs ); + // populate missing informations + PDCs::populateMissingInformations( $pdcs ); // sort by creation date PDCs::sortByCreationDate( $pdcs ); // index then by their PDC_TYPE $pdcs_by_type = PDCs::indexByType( $pdcs ); // save the counting page PageYearMonthDayPDCsCount::createFromDateTimePDCs( $this->getDate(), $pdcs_by_type ) ->save(); // save the log page PageYearMonthDayPDCsLog::createFromDateTimePDCs( $this->getDate(), $pdcs_by_type ) ->save(); return $this; } } diff --git a/includes/class-PDCs.php b/includes/class-PDCs.php index 5d450a0..2ec8aed 100644 --- a/includes/class-PDCs.php +++ b/includes/class-PDCs.php @@ -1,129 +1,162 @@ . namespace itwikidelbot; use DateTime; -use wm\WikipediaIt; use mw\API\PageMatcher; /** * Handle some PDCs */ class PDCs { /** * Sort some PDCs by creation date * * @param $pdcs array */ public static function sortByCreationDate( & $pdcs ) { // sort PDCs by start date usort( $pdcs, function ( $a, $b ) { return $a->getCreationDate() > $b->getCreationDate(); } ); } /** * Index some PDCs by their type * * @param $pdcs array * @return array */ public static function indexByType( $pdcs ) { $pdcs_by_type = []; foreach( CategoryYearMonthDayTypes::all() as $Type ) { $pdcs_by_type[ $Type::PDC_TYPE ] = []; } foreach( $pdcs as $pdc ) { $type = $pdc->getType(); $pdcs_by_type[ $type ][] = $pdc; } return $pdcs_by_type; } /** * Discard all the PDCs that do not belong to a certain date * * @param $pdcs array * @param $date DateTime * @return array */ public static function filterByDate( $pdcs, DateTime $date ) { $y_m_d = $date->format( 'Y-m-d' ); return array_filter( $pdcs, function ( $pdc ) use ( $y_m_d ) { return $pdc->getStartDate()->format( 'Y-m-d' ) === $y_m_d; } ); } /** * Discard all the PDCs that are not multiple * * @param $pdcs array * @return $pdcs */ public static function filterNotMultiple( $pdcs ) { return array_filter( $pdcs, function ( $pdc ) { return ! $pdc->isMultiple(); } ); } /** - * Populate the PDC 'subject themes' field of some PDCs + * Get all the title subjects of the specified PDCs + * + * @param $pdcs array + * @return array + */ + public static function titleSubjects( $pdcs ) { + return array_map( function ( $pdc ) { + return $pdc->getTitleSubject(); + }, $pdcs ); + } + + /** + * Populate the specified PDCs that miss informations * * This method is intended to do less API requests as possible. * * @param $pdcs array * @return array - * @see PDC::getSubjectThemes() */ - public static function populateSubjectThemes( $pdcs ) { - - // only non multiple PDCs - $pdcs = self::filterNotMultiple( $pdcs ); + public static function populateMissingInformations( $pdcs ) { + self::populateMissingInformationsFromLastRevision( $pdcs ); + } - // subject titles - $titles = array_map( function ( $pdc ) { - return $pdc->getTitleSubject(); - }, $pdcs ); + /** + * Populate the specified PDCs with missing informations that can + * be obtained fetching the last revision. + * + * This method is intended to do less API requests as possible. + * + * @param $pdcs + */ + private static function populateMissingInformationsFromLastRevision( $pdcs ) { - // API to retrieve the wikitext in the first section of all the pages - $site = WikipediaIt::getInstance(); - $query = $site->createQuery( [ + /* + * fetch the last revision from all the PDCs + * + * @see https://it.wikipedia.org/w/api.php?action=help&modules=query%2Brevisions + */ + $query = Page::api()->createQuery( [ 'action' => 'query', - 'titles' => $titles, + 'titles' => self::titleSubjects( $pdcs ), 'prop' => 'revisions', - 'rvprop' => 'content', + 'rvprop' => [ + 'content', + 'timestamp', + ], 'rvsection' => '0', ] ); + // callback fired for every match between response pages and PDCs + $matching_callback = function ( $page, $pdc ) { + if( isset( $page->revisions, $page->revisions[ 0 ] ) ) { + + // this is the only one + $revision = $page->revisions[ 0 ]; + + // populate the subject themes + $pdc->setSubjectThemesScrapingSubjectWikitext( $revision->{ '*' } ); + + // populate the lastedit date + $lastedit_date = self::createDateTimeFromString( $revision->timestamp ); + $pdc->setLasteditDate( $lastedit_date ); + } + }; + + // callback that returns the PDC page title + $pdc_page_title_callback = function ( $pdc ) { + return $pdc->getTitleSubject(); + }; + + // query continuation foreach( $query->getGenerator() as $response ) { - ( new PageMatcher( $response->query, $pdcs ) )->matchByTitle( - // callback fired for every match between response pages and PDCs - function ( $page, $pdc ) { - if( isset( $page->revisions ) ) { - $page_content = $page->revisions[ 0 ]->{ '*' }; - $pdc->setSubjectThemesScrapingSubjectWikitext( $page_content ); - } - }, - // callback that must returns the PDC page title - function ( $pdc ) { - return $pdc->getTitleSubject(); - } - ); + // match page results and PDCs by title + ( new PageMatcher( $response->query, $pdcs ) ) + ->matchByTitle( $matching_callback, $pdc_page_title_callback ); } } } + diff --git a/includes/class-Page.php b/includes/class-Page.php index 0699b38..03e2232 100644 --- a/includes/class-Page.php +++ b/includes/class-Page.php @@ -1,230 +1,230 @@ . namespace itwikidelbot; use DateTime; use DateTimeZone; use cli\Input; use cli\Log; use wm\WikipediaIt; use mw\Tokens; /** * Handle a page */ class Page { /** * Time zone of Italian Wikipedia community. * * @var string */ const COMMUNITY_TIMEZONE = 'Europe/Rome'; /** * Enable this flag to ask for every changes * * @var bool */ public static $ASK_BEFORE_SAVING = false; /** * @var string Page title with prefix */ private $title; /** * Construct a Page * * @param $title Page title with its prefix */ public function __construct( $title ) { $this->title = $title; } /** * Get the page title with its prefix * * @return string */ public function getTitle() { return $this->title; } /** * Save this page * * @param $content string Page content * @param $summary string Edit summary * @return bool|mixed False if not created */ public function saveByContentSummary( $content, $summary ) { return static::saveByTitleContentSummary( $this->getTitle(), $content, $summary ); } /** * Save this page if it does not exist * * @param $content string Page content * @param $summary string Edit summary * @return bool|mixed False if not created */ public function saveByContentSummaryIfNotExists( $content, $summary ) { return self::saveByTitleContentSummaryIfNotExists( $this->getTitle(), $content, $summary ); } /** * Check if this page exists * * @return bool */ public function exists() { return static::existsByTitle( $this->getTitle() ); } /** * Save a page * * @param $title string Page name with prefix * @param $content string Page content * @param $summary string Edit summary * @return mixed Response */ public static function saveByTitleContentSummary( $title, $content, $summary ) { $api = self::api(); $args = [ 'action' => 'edit', 'title' => $title, 'text' => $content, 'summary' => $summary, 'token' => $api->login()->getToken( Tokens::CSRF ), 'bot' => 1, ]; if( self::$ASK_BEFORE_SAVING ) { print_r( $args ); if( 'y' !== Input::yesNoQuestion( "Save?" ) ) { return false; } } Log::info( "writing [[$title]]" ); return $api->post( $args ); } /** * Fetch the first revision date by direction * * @param $dir string direction * @return DateTime */ public function fetchFirstRevisionDateByDirection( $direction ) { $response = self::api()->fetch( [ 'action' => 'query', 'titles' => $this->getTitle(), 'prop' => 'revisions', 'rvprop' => [ 'timestamp' ], 'rvlimit' => 1, 'rvdir' => $direction, ] ); foreach( $response->query->pages as $page ) { if( isset( $page->revisions ) ) { foreach( $page->revisions as $revision ) { return self::createDateTimeFromString( $revision->timestamp ); } } } throw new \Exception( 'unable to fetch the creation date' ); } /** * Fetch the creation date of this page * * @return DateTime */ public function fetchCreationDate() { return $this->fetchFirstRevisionDateByDirection( 'newer' ); } /** * Fetch the creation date of this page * * @return DateTime */ public function fetchLasteditDate() { return $this->fetchFirstRevisionDateByDirection( 'older' ); } /** * Save a page if it does not exist * * @param $title string Page name with prefix * @param $content string Page content * @param $summary string Edit summary * @return bool|mixed False if not created */ public static function saveByTitleContentSummaryIfNotExists( $title, $content, $summary ) { if( ! static::existsByTitle( $title ) ) { return static::saveByTitleContentSummary( $title, $content, $summary ); } return false; } /** * Check if a page title exists * * @param $title string Page name with prefix * @return bool */ public static function existsByTitle( $title ) { $result = self::api()->fetch( [ 'action' => 'query', 'prop' => 'info', 'titles' => $title, ] ); foreach( $result->query->pages as $pageid => $page ) { return ! isset( $page->missing ); } return false; } /** * Get the API related to this page * * @return mw\API */ - protected static function api() { + public static function api() { return WikipediaIt::getInstance(); } /** * Create a DateTime object from a MediaWiki formatted date * * MediaWiki dates are formatted following the ISO8601 standard * and you may want to specify your community timezone. * * @param $datetime string * @return DateTime */ public static function createDateTimeFromString( $datetime ) { return DateTime::createFromFormat( DateTime::ISO8601, $datetime ) ->setTimezone( new DateTimeZone( self::COMMUNITY_TIMEZONE ) ); } }