Advertisement
Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- <?php
- ##http://proxy-base.com/208173-post6.html
- require "vendor/autoload.php";
- use Amp\Http\Client\HttpClientBuilder;
- use Amp\Http\Client\Request;
- use Amp\Loop;
- use Symfony\Component\DomCrawler\Crawler;
- $from = new DateTime('2006-06-01');
- $to = new DateTime('2020-05-02');
- Loop::run(function () use (&$from, $to) {
- $client = HttpClientBuilder::buildDefault();
- $handle = \Amp\File\open("domains.txt", "w");
- while ($from < $to) {
- $uri = "https://whoistory.com/" . $from->format('/Y/m/d/');
- $response = yield $client->request(new Request($uri, 'GET'));
- if ($response->getStatus() == 200){
- $crawler = new Crawler((string) yield $response->getBody()->buffer());
- $links = $crawler->filter('div.left > a');
- $links->each(function ($node) use ($handle) {
- if(substr($node->attr('href'), 0, -1) != null && $node->attr('class') != "backlink") {
- $handle->onResolve(function ($error, $result) use ($node) {
- if ($error !== null) {
- exit($error->getMessage());
- }
- $write = $result->write($node->text() . "\n");
- });
- }
- });
- echo $uri . ' : ' . $links->count() . PHP_EOL;
- } else {
- echo $uri . " 404" . PHP_EOL;
- }
- $from->modify("+ 1 day");
- }
- });
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement