diff --git "a/class-mw\\API.php" "b/class-mw\\API.php" new file mode 100644 index 0000000..19fc88a --- /dev/null +++ "b/class-mw\\API.php" @@ -0,0 +1,215 @@ +. + +# MediaWiki +namespace mw; + +/** + * Make HTTP request to a MediaWiki API + */ +class API extends \network\HTTPRequest { + + /** + * Default MediaWiki login username + * + * @var string + */ + static $DEFAULT_USERNAME; + + /** + * Default MediaWiki login password + * + * @var string + */ + static $DEFAULT_PASSWORD; + + /** + * Default MediaWiki API maxlag + * + * @var int + */ + static $DEFAULT_MAXLAG = 5; + + /** + * Default MediaWiki API result format + * + * @var string + */ + static $DEFAULT_FORMAT = 'json'; + + /** + * Frequent API response token to continue a result set. + * + * @var mixed + */ + private $continue = null; + + /** + * Flag to avoid duplicate logins + */ + private $logged = false; + + /** + * Last API response + * + * @var mixed + */ + private $last = null; + + /** + * Effectuate an HTTP POST request but only after a login. + * + * @param $data array GET/POST data + * @param $args array Internal arguments + * @override \network\HTTPRequest#post() + * @return mixed + */ + public function post( $data = [], $args = [] ) { + if( ! $this->isLogged() ) { + $this->login(); + } + return parent::post( $data, $args ); + } + + /** + * Check if there are other requests to do to fetch the whole result set. + * + * @return bool + */ + public function hasNext() { + // Note that as default continue is NULL + return false !== $this->continue; + } + + /** + * Fetch the next result set. + * + * @return mixed + */ + public function fetchNext() { + $data = $this->getData(); + + if( $this->continue ) { + self::log('INFO', "Will continue"); + foreach( $this->continue as $arg => $value ) { + $data[ $arg ] = $value; + } + } + + $next = $this->fetch( $data ); + + $this->continue = isset( $next->continue ) + ? $next->continue + : false; + + return $next; + } + + /** + * Check if it's already logged in. + * + * @return bool + */ + public function isLogged() { + return $this->logged; + } + + /** + * Login into MediaWiki using an username/password pair. + * + * Yes, I'm talking about a bot password. + */ + public function login( $username = null, $password = null ) { + + // Can use a default set of credentials + if( ! $username && ! $password ) { + $username = self::$DEFAULT_USERNAME; + $password = self::$DEFAULT_PASSWORD; + } + + if( ! $username || ! $password ) { + throw new \Exception( sprintf( + 'you must call %1$s#login( $username, $password ) or '. + 'set %1$s::$DEFAULT_USERNAME and %1$s::$DEFAULT_PASSWORD ' . + 'before POSTing', + __CLASS__ + ) ); + } + + // Fetch the login token + $logintoken = $this->fetch( [ + 'action' => 'query', + 'meta' => 'tokens', + 'type' => 'login' + ] )->query->tokens->logintoken; + + // Login + $response = parent::post( [ + 'action' => 'login', + 'lgname' => $username, + 'lgpassword' => $password, + 'lgtoken' => $logintoken + ] ); + if( ! isset( $response->login->result ) || $response->login->result !== 'Success' ) { + throw new \Exception("login failed"); + } + + return $this; + } + + /** + * Set GET/POST data. + * + * As default, it propose a maxlag and a format. + * + * @param $data array GET/POST data + * @overload \network\HTTPRequest#setData() + */ + public function setData( $data ) { + $this->continue = null; + $data = array_replace( [ + 'maxlag' => self::$DEFAULT_MAXLAG, + 'format' => self::$DEFAULT_FORMAT + ], $data ); + return parent::setData( $data ); + } + + /** + * Check formal API errors + * + * @param $result mixed Result + * @override \network\HTTPRequest#onFetched() + */ + protected function onFetched( $result ) { + if( isset( $result->error ) ) { + // Retry after some time when server lags + if( 'maxlag' === $result->error->code ) { + self::log( 'WARN', "Lag!" ); + $args = array_replace( [ + 'wait' => self::WAIT_DOS + ], $args ); + $result = $this->fetch( $data , $args ); + } else { + // Invalid query? + print_r( $result->error ); + throw new \Exception( "API error" ); + } + } + $this->last = $result; + return $result; + } +} diff --git "a/class-mw\\Site.php" "b/class-mw\\Site.php" index f9de59b..6854be5 100644 --- "a/class-mw\\Site.php" +++ "b/class-mw\\Site.php" @@ -1,91 +1,96 @@ . # MediaWiki namespace mw; class Site { private $api; /** * @var Namespace[] */ private $namespaces = []; public function __construct( $url ) { $this->setApi( $url ); } public function getApi() { return $this->api; } public function setApi( $url ) { $this->api = self::createApi( $url ); return $this; } public function fetch( $data ) { return $this->getApi()->fetch( $data ); } public function post( $data ) { return $this->getApi()->post( $data ); } + public function login( $username = null, $password = null ) { + $this->getApi()->login( $username, $password ); + return $this; + } + public function hasNext() { return $this->getApi()->hasNext(); } public function fetchNext() { return $this->getApi()->fetchNext(); } public function setApiData( $data ) { $this->getApi()->setData( $data ); return $this; } public function hasNamespace( $id ) { return array_key_exists( $id, $this->namespaces ); } public function getNamespace( $id ) { if( ! $this->hasNamespace( $id ) ) { throw new \Exception( sprintf( 'missing namespace %d', $id ) ); } return $this->namespaces[ $id ]; } public function setNamespace( Ns $namespace ) { $id = $namespace->getID(); $this->namespaces[ $id ] = $namespace; return $this; } public function setNamespaces( $namespaces ) { foreach( $namespaces as $namespace ) { $this->setNamespace( $namespace ); } return $this; } private static function createApi( $url ) { - return new APIRequest( $url ); + return new API( $url ); } } diff --git "a/class-mw\\APIRequest.php" "b/class-network\\HTTPRequest.php" similarity index 63% rename from "class-mw\\APIRequest.php" rename to "class-network\\HTTPRequest.php" index a479d64..63f46aa 100644 --- "a/class-mw\\APIRequest.php" +++ "b/class-network\\HTTPRequest.php" @@ -1,301 +1,433 @@ . # MediaWiki -namespace mw; +namespace network; -class APIRequest { - static $WAIT = 0.2; +/** + * HTTP request handler for GET and POST requests. + */ +class HTTPRequest { + + /** + * Class version + * + * To be incremented every time do you notice this number. + * + * @var string + */ + const VERSION = 0.4; + + /** + * Source code URL + * + * @var string + */ + const REPO = 'https://github.com/valerio-bozzolan/boz-mw'; + + /** + * Wait seconds before each GET request. + */ + static $WAIT = 0.2; + + /** + * Wait seconds before each POST request + * + * @var float + */ static $WAIT_POST = 0.2; + /** + * Wait seconds before each lag + * + * @var float + */ const WAIT_DOS = 5; - const VERSION = 0.3; - + /** + * Full HTTP URL to the API endpoint + * + * @var string + */ private $api; + + /** + * HTTP GET/POST data + * + * @var array + */ private $data; - private $args; - private $continue; - private $last = null; + /** + * Internal arguments + * + * @param array + */ + private $args; + /** + * Latest HTTP response headers + * + * @var array + */ private $latestHttpResponseHeaders = []; + + /** + * HTTP cookies + * + * @var array + */ private $cookies = []; + /** + * Flag for debug mode + * + * @var bool + */ private $debug = false; + /** + * Constructor. + * + * @param $api string API endpoint + * @param $data array GET/POST data + * @param $args array Internal arguments + */ public function __construct( $api, $data = [], $args = [] ) { $this->api = $api; $this->setData( $data ); $this->setArgs( $args ); } - public function setData( $data ) { - $this->data = array_replace( [ - 'maxlag' => 5, - 'format' => 'json' - ], $data ); - $this->continue = null; - return $this; - } - - public function setArgs( $args ) { - $this->args = array_replace( [ - 'method' => 'GET', - 'wait' => self::$WAIT, - 'wait-post' => self::$WAIT_POST, - 'user-agent' => sprintf( 'Boz-MW APIRequest.php/%s', self::VERSION ), - 'headers' => [], - 'assoc' => false - ], $args ); - return $this; + /** + * Statical constructor. + * + * @return network\HTTPRequest + */ + public static function factory( $api, $data = [], $args = [] ) { + return new self( $api, $data, $args ); } - public static function factory( $api, $args = [] ) { - return new self( $api, $args ); + /** + * Get current HTTP GET/POST data. + * + * @return array + */ + public function getData() { + return $this->data; } - public function lastfetch() { - return isset( $this->last ) ? $this->last : $this->fetch(); + /** + * Get internal arguments. + * + * @return array + */ + public function getArgs() { + return $this->args; } - // https://www.wikidata.org/w/api.php?action=wbgetclaims&entity=Q38&format=jsonfm - public function fetchFirstClaimValue($property) { - return @ $this->lastfetch()->claims->{$property}[0]->mainsnak->datavalue->value; + /** + * Check if debug mode is enabled + * + * @return bool + */ + public function isDebug() { + return $this->debug; } - public function hasNext() { - return $this->continue !== false; + /** + * Set HTTP GET/POST data for same future requests. + * + * @param $data array GET/POST data + * @return self + */ + public function setData( $data ) { + $this->data = $data; + return $this; } + /** + * Toggle debug mode + * + * @return self + */ public function setDebug( $debug ) { $this->debug = $debug; return $this; } + /** + * Set internal arguments + * + * @param $args array Internal arguments + * @return self + */ + public function setArgs( $args ) { + $this->args = array_replace( [ + 'method' => 'GET', + 'wait' => self::$WAIT, + 'wait-post' => self::$WAIT_POST, + 'user-agent' => sprintf( 'boz-mw HTTPRequest.php/%s %s', self::VERSION, self::REPO ), + 'headers' => [], + 'assoc' => false + ], $args ); + return $this; + } + + /** + * Make an HTTP GET query + * + * @param $data array GET data + * @param $args array Internal arguments + * @return mixed Response + */ public function fetch( $data = [], $args = [] ) { $data = array_replace( $this->data, $data ); $args = array_replace( $this->args, $args ); // HTTP query using file_get_contents() $url = $this->api; $query = http_build_query( $data ); $context = [ 'http' => [] ]; $context['http']['method'] = $args['method']; if( $args['user-agent'] ) { $args['headers'][] = self::header( 'User-Agent', $args['user-agent'] ); } if( $this->haveCookies() ) { $args['headers'][] = $this->getCookieHeader(); } if( $query ) { if( 'POST' === $args['method'] ) { $args['headers'][] = self::header('Content-Type', 'application/x-www-form-urlencoded' ); $context['http']['content'] = $query; } else { $url .= "?$query"; } } if( $args['headers'] ) { $context['http']['header'] = self::implodeHTTPHeaders( $args['headers'] ); } - if( $this->debug ) { + if( $this->isDebug() ) { self::log('DEBUG', $url ); } if( $args['wait'] ) { self::log('INFO', sprintf( "Waiting %.2f s...", $args['wait'] ) ); usleep( $args['wait'] * 1000000 ); } self::log('INFO', sprintf( "%s request", $args['method'] ) ); $stream_context = stream_context_create( $context ); $result = file_get_contents( $url, false, $stream_context ); $result = json_decode( $result, $args['assoc'] ); // Here $http_response_header exists magically (PHP merda!) $this->loadHTTPResponseHeaders( $http_response_header ); if( isset( $this->latestHttpResponseHeaders['HTTP/1.1 500 Internal Server Error'] ) ) { self::log( 'WARN', "500 Internal Server Error!"); $args = array_replace( [ 'wait' => self::WAIT_DOS ], $args ); return $this->fetch( $data , $args ); } self::log('INFO', "Fetched"); - if( isset( $result->error ) ) { - if( 'maxlag' === $result->error->code ) { - self::log( 'WARN', "Lag!"); - $args = array_replace( [ - 'wait' => self::WAIT_DOS - ], $args ); - return $this->fetch( $data , $args ); - } else { - print_r( $result->error ); - throw new \Exception( "API error" ); - } - } - - return $this->last = $result; + return $this->onFetched( $result ); } + /** + * Effectuate an HTTP POST. + */ public function post( $data = [], $args = [] ) { $args = array_replace( [ 'method' => 'POST', 'wait' => $this->args['wait-post'] ], $args ); return $this->fetch( $data, $args ); } /** - * @deprecated + * Get latest HTTP response headers. + * + * @return array */ - public function getNext() { - return $this->fetchNext(); - } - - public function fetchNext() { - $data = $this->data; - - if( $this->continue ) { - self::log('INFO', "Will continue"); - foreach($this->continue as $arg => $value) { - $data[ $arg ] = $value; - } - } - - $next = $this->fetch( $data ); - - $this->continue = isset( $next->continue ) - ? $next->continue - : false; - - return $next; - } - public function getLatestHTTPResponseHeaders() { return $this->latestHttpResponseHeaders; } /** - * Load HTTP response headers, filling cookies. + * Check if cookies are set + * + * @return bool */ - private function loadHTTPResponseHeaders( $http_response_headers ) { - $this->latestHttpResponseHeaders = self::parseHTTPResponseHeaders( $http_response_headers ); - if( isset( $this->latestHttpResponseHeaders['Set-Cookie'] ) ) { - foreach( $this->latestHttpResponseHeaders['Set-Cookie'] as $cookie ) { - $this->setRawCookie( $cookie ); - } - } + public function haveCookies() { + return isset( $this->cookies ); } /** - * Group HTTP headers by keys. + * Get cookies * * @return array */ - private static function parseHTTPResponseHeaders( $http_response_headers ) { - $headers = []; - foreach( $http_response_headers as $header ) { - $header_parts = explode(':', $header, 2); - if( 2 === count( $header_parts ) ) { - list( $name, $value ) = $header_parts; - if( ! isset( $headers[ $name ] ) ) { - $headers[ $name ] = []; - } - $headers[ $name ][] = ltrim( $value ); - } - } - return $headers; - } - - public function haveCookies() { - return $this->cookies; - } - public function getCookies() { return $this->cookies; } + /** + * Set an HTTP cookie name => value pair. + * + * @param $name string Cookie name + * @param $value string Cookie value + * @return self + */ public function setCookie( $name, $value ) { $this->cookies[ $name ] = $value; return $this; } - private function setRawCookie( $cookie ) { - $parts = explode(';', $cookie); - if( isset( $parts[0] ) ) { - $name_value = explode('=', $parts[0] ); - if( 2 === count( $name_value ) ) { - list( $name, $value ) = $name_value; - $this->setCookie( $name, $value ); - } - } - } - public function getCookieHeader() { $cookies = []; foreach( $this->getCookies() as $name => $value ) { $cookies[] = urlencode( $name ) . '=' . urlencode( $value ); } return self::headerRaw( 'Cookie', implode( '; ', $cookies ) ); } public function getHTTPCookies() { return $this->cookies; } + /** + * Set an HTTP cookie in its raw form. + * + * @param $cookie string Cookie text + */ + private function setRawCookie( $cookie ) { + $parts = explode(';', $cookie); + if( isset( $parts[0] ) ) { + $name_value = explode('=', $parts[0] ); + if( 2 === count( $name_value ) ) { + list( $name, $value ) = $name_value; + $this->setCookie( $name, $value ); + } + } + } + + /** + * Load HTTP response headers, filling cookies. + */ + private function loadHTTPResponseHeaders( $http_response_headers ) { + $this->latestHttpResponseHeaders = self::parseHTTPResponseHeaders( $http_response_headers ); + if( isset( $this->latestHttpResponseHeaders['Set-Cookie'] ) ) { + foreach( $this->latestHttpResponseHeaders['Set-Cookie'] as $cookie ) { + $this->setRawCookie( $cookie ); + } + } + } + private function implodeHTTPHeaders( $headers ) { if( ! $headers ) { return null; } $s = ''; foreach( $headers as $header ) { $s .= "$header\r\n"; } return $s; } + /** + * Group HTTP headers by keys. + * + * @return array + */ + private static function parseHTTPResponseHeaders( $http_response_headers ) { + $headers = []; + foreach( $http_response_headers as $header ) { + $header_parts = explode(':', $header, 2); + if( 2 === count( $header_parts ) ) { + list( $name, $value ) = $header_parts; + if( ! isset( $headers[ $name ] ) ) { + $headers[ $name ] = []; + } + $headers[ $name ][] = ltrim( $value ); + } + } + return $headers; + } + + /** + * Make an HTTP header raw string from a name => value pair. + * + * The value will be URL-encoded. + * + * @param $name string HTTP header name + * @param $value string HTTP header value + * @return string HTTP header + */ private static function header( $name, $value ) { return self::headerRaw( $name, urlencode( $value ) ); } + /** + * Make an HTTP header raw string from a name => value pair. + * + * The value is not URL-encoded. + * + */ private static function headerRaw( $name, $value ) { return sprintf( '%s: %s', $name, $value ); } + /** + * Print a tedious message to the output. + * + * @param $type string Something like 'warn' + * @param $msg string Log message + */ private static function log( $type, $msg ) { printf("# [%s] \t %s\n", $type, $msg); } + + /** + * Callback to be overloaded + * + * This is called every time something is fetched. + * + * @param $result mixed Result + * @return mixed Result + */ + protected function onFetched( $result ) { + return $result; + } }