Compare commits

..

No commits in common. "main" and "feature/safti/20" have entirely different histories.

13 changed files with 1963 additions and 3405 deletions

View File

@ -1,16 +0,0 @@
kind: pipeline
name: default
steps:
- name: install
pull: if-not-exists
image: composer
commands:
- cp .env.example .env
- composer install --ignore-platform-reqs --prefer-dist
- php artisan key:generate
- name: test
image: php:7
commands:
- vendor/bin/phpunit --configuration phpunit.xml

View File

@ -2,8 +2,6 @@
namespace App\Exceptions; namespace App\Exceptions;
use Exception;
class UnknownParser extends Exception class UnknownParser extends Exception
{ {

View File

@ -73,22 +73,20 @@ class Home extends Model implements Feedable
{ {
// Download images on creation // Download images on creation
static::created(static function ($home) { static::created(static function ($home) {
if (!empty($home->pictures)) {
Artisan::call('app:downloadimage ' . $home->id); Artisan::call('app:downloadimage ' . $home->id);
}
}); });
} }
/** /**
* @inheritDoc * @inheritDoc
*/ */
public function toFeedItem(): FeedItem public function toFeedItem()
{ {
return FeedItem::create() return FeedItem::create()
->id($this->id) ->id($this->id)
->title($this->title) ->title($this->title)
->summary($this->excerpt()) ->summary($this->excerpt())
->authorName('Shikiryu') ->author('Shikiryu')
->updated($this->created_at) ->updated($this->created_at)
->link(\route('public.view', ['slug' => $this->slug])) ->link(\route('public.view', ['slug' => $this->slug]))
; ;

View File

@ -27,7 +27,6 @@ abstract class Parser
* @param string $url * @param string $url
* *
* @return \App\Parser|null * @return \App\Parser|null
* @throws \App\Exceptions\UnknownParser
*/ */
public static function factory(string $url): ?Parser public static function factory(string $url): ?Parser
{ {
@ -49,75 +48,4 @@ abstract class Parser
* @return \App\ParsedHome * @return \App\ParsedHome
*/ */
abstract public function parse(): ParsedHome; abstract public function parse(): ParsedHome;
/**
* @param int $score
*
* @return string
*/
protected function calculateDPE($score)
{
if (empty($score)) {
return 'Inconnu';
}
if ($score <= 50) {
return 'A';
}
if ($score >= 51 && $score <= 90) {
return 'B';
}
if ($score >= 91 && $score <= 150) {
return 'C';
}
if ($score >= 151 && $score <= 230) {
return 'D';
}
if ($score >= 231 && $score <= 330) {
return 'E';
}
if ($score >= 331 && $score <= 450) {
return 'F';
}
if ($score > 450) {
return 'G';
}
return 'Inconnu';
}
/**
* @param $score
*
* @return string
*/
protected function calculateGES($score)
{
if (empty($score)) {
return 'Inconnu';
}
if ($score <= 5) {
return 'A';
}
if ($score >= 6 && $score <= 10) {
return 'B';
}
if ($score >= 11 && $score <= 20) {
return 'C';
}
if ($score >= 21 && $score <= 35) {
return 'D';
}
if ($score >= 36 && $score <= 55) {
return 'E';
}
if ($score >= 56 && $score <= 80) {
return 'F';
}
if ($score > 80) {
return 'G';
}
return 'Inconnu';
}
} }

View File

@ -36,6 +36,76 @@ class Orpi extends Parser
return $this->parseHTML($parsedHome, $crawler); return $this->parseHTML($parsedHome, $crawler);
} }
/**
* @param int $score
*
* @return string
*/
private function calculateDPE($score)
{
if (empty($score)) {
return 'Inconnu';
}
if ($score <= 50) {
return 'A';
}
if ($score >= 51 && $score <= 90) {
return 'B';
}
if ($score >= 91 && $score <= 150) {
return 'C';
}
if ($score >= 151 && $score <= 230) {
return 'D';
}
if ($score >= 231 && $score <= 330) {
return 'E';
}
if ($score >= 331 && $score <= 450) {
return 'F';
}
if ($score > 450) {
return 'G';
}
return 'Inconnu';
}
/**
* @param $score
*
* @return string
*/
private function calculateGES($score)
{
if (empty($score)) {
return 'Inconnu';
}
if ($score <= 5) {
return 'A';
}
if ($score >= 6 && $score <= 10) {
return 'B';
}
if ($score >= 11 && $score <= 20) {
return 'C';
}
if ($score >= 21 && $score <= 35) {
return 'D';
}
if ($score >= 36 && $score <= 55) {
return 'E';
}
if ($score >= 56 && $score <= 80) {
return 'F';
}
if ($score > 80) {
return 'G';
}
return 'Inconnu';
}
/** /**
* @param \App\ParsedHome $parsed_home * @param \App\ParsedHome $parsed_home
* @param \Symfony\Component\DomCrawler\Crawler $crawler * @param \Symfony\Component\DomCrawler\Crawler $crawler

View File

@ -18,13 +18,9 @@ class OuestFrance extends Parser
$crawler = new Crawler($body); $crawler = new Crawler($body);
$parsedHome = new ParsedHome(); $parsedHome = new ParsedHome();
$parsedHome->title = $crawler->filter('h1')->text(); $parsedHome->title = $crawler->filter('h1')->text();
$details = $crawler->filter('#blocCaractAnn > ul > li'); $parsedHome->price = (int)str_replace(' ', '', $crawler->filter('#strongPrix')->text());
$price = $crawler->filter('#strongPrix');
if (!$price instanceof Crawler) {
$price = $details->eq(0)->filter('strong')->text();
}
$parsedHome->price = (int)str_replace(' ', '', $price->text());
$parsedHome->description = $crawler->filter('#blockonDescriptif')->text(); $parsedHome->description = $crawler->filter('#blockonDescriptif')->text();
$details = $crawler->filter('#blocCaractAnn > ul > li');
$parsedHome->surface = (int)str_replace(' ', '', $details->eq(2)->filter('strong')->text()); $parsedHome->surface = (int)str_replace(' ', '', $details->eq(2)->filter('strong')->text());
$parsedHome->garden_surface = (int)str_replace(' ', '', $details->eq(3)->filter('strong')->text()); $parsedHome->garden_surface = (int)str_replace(' ', '', $details->eq(3)->filter('strong')->text());
$parsedHome->rooms = (int)str_replace(' ', '', $details->eq(4)->filter('strong')->text()); $parsedHome->rooms = (int)str_replace(' ', '', $details->eq(4)->filter('strong')->text());

View File

@ -27,17 +27,14 @@ class Safti extends Parser
$currency = 'EUR'; $currency = 'EUR';
$parsed_home->title = $property_single->filter('h1')->text(); $parsed_home->title = $property_single->filter('h1')->text();
$parsed_home->price = $currency_formatter $parsed_home->price = $currency_formatter->parseCurrency($property_single->filter('.property__price')->text(), $currency);
->parseCurrency($property_single->filter('.property__price')->text(), $currency); $parsed_home->city = $property_single->children()->children('div')->eq(1)->filter('p.h4')->text();
$parsed_home->city = $property_single?->children()?->children('div')?->eq(0)?->filter('p.h4')?->text();
$parsed_home->description = $crawler->filter('[data-testid="real-estate-annonce-single-description"]')->text(); $parsed_home->description = $crawler->filter('[data-testid="real-estate-annonce-single-description"]')->text();
$property__additionals = $crawler->filter('.property__additionals'); $property__additionals = $crawler->filter('.property__additionals');
$energies = $property__additionals->filter('.energetic-indicator'); $energies = $property__additionals->filter('.energetic-indicator');
if ($energies->count() >= 1) { if ($energies->count() > 0) {
$parsed_home->energy = substr($energies->eq(0)->text(), 0, 1); $parsed_home->energy = substr($energies->eq(0)->text(), 0, 1);
}
if ($energies->count() >= 2) {
$parsed_home->ges = substr($energies->eq(1)->text(), 0, 1); $parsed_home->ges = substr($energies->eq(1)->text(), 0, 1);
} }
@ -61,10 +58,7 @@ class Safti extends Parser
} }
}); });
$parsed_home->pictures = $crawler $parsed_home->pictures = $crawler->filter('[data-testid="real-estate-mosaic-photo"]')->filter('img')->each(static function($img) {
->filter('[data-testid="real-estate-mosaic-photo"]')
->filter('img')
->each(static function ($img) {
return $img->attr('src'); return $img->attr('src');
}); });

View File

@ -4,8 +4,6 @@ namespace App\Parser;
use App\ParsedHome; use App\ParsedHome;
use App\Parser; use App\Parser;
use GuzzleHttp\Exception\GuzzleException;
use Symfony\Component\DomCrawler\Crawler;
/** /**
* Thanks to https://github.com/axeleroy/untoitpourcaramel/issues/3 * Thanks to https://github.com/axeleroy/untoitpourcaramel/issues/3
@ -27,7 +25,6 @@ class SeLoger extends Parser
$token = $this->retrieveToken(); $token = $this->retrieveToken();
$idAnnonce = $this->getIdAnnonceFromUrl($this->url); $idAnnonce = $this->getIdAnnonceFromUrl($this->url);
$url = sprintf('%s%s', self::LISTING_URL, $idAnnonce); $url = sprintf('%s%s', self::LISTING_URL, $idAnnonce);
try {
$request = $this->client->request( $request = $this->client->request(
'GET', 'GET',
$url, $url,
@ -37,9 +34,6 @@ class SeLoger extends Parser
], ],
] ]
); );
} catch (GuzzleException $e) {
return $this->parseHTML();
}
$annonce = json_decode($request->getBody()->getContents(), true); $annonce = json_decode($request->getBody()->getContents(), true);
$parsedHome = new ParsedHome(); $parsedHome = new ParsedHome();
@ -89,61 +83,4 @@ class SeLoger extends Parser
return trim($request->getBody()->getContents(), '"'); return trim($request->getBody()->getContents(), '"');
} }
/**
* @return \App\ParsedHome
*/
private function parseHTML()
{
$request = $this->client->get(
$this->url,
[
'headers' => [
'User-Agent' => 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:85.0) Gecko/20100101 Firefox/85.0',
'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
'Accept-Encoding' => 'gzip, deflate, br',
'Accept-Language' => 'fr,fr-FR;q=0.8,en-US;q=0.5,en;q=0.3',
'Cache-Control' => 'no-cache',
'Connection' => 'keep-alive',
]
]
);
$body = $request->getBody()->getContents();
$crawler = new Crawler($body);
$parsed_home = new ParsedHome();
$parsed_home->title = $crawler->filter('h1')->text();
$parsed_home->description = $crawler->filter('#showcase-description > div')->first()->text();
$parsed_home->price = (int) str_replace(' ', '', $crawler->filter('[class^=Summarystyled__PriceText]')->text());
$parsed_home->city = $crawler->filter('[class^=Summarystyled__Address]')->text();
$parsed_home->energy = $this->calculateDPE(
(int)$crawler->filter('[class^=Preview__PreviewTooltipValue]')->first()->text()
);
$parsed_home->ges = $this->calculateGES(
(int)$crawler->filter('[class^=Preview__PreviewTooltipValue]')->eq(1)->text()
);
$crawler
->filter('[class^=Summarystyled__TagsWrapper] > div')
->each(static function (Crawler $property____information, $i) use (&$parsed_home) {
$element = $property____information->children()->eq(1)->text();
if ('pièces' === mb_substr($element, -6)) {
$parsed_home->rooms = (int) $element;
}
if ('m²' === mb_substr($element, -2) && strpos($element, '/') === false) {
$parsed_home->surface = (int) $element;
}
});
$parsed_home->pictures = $crawler
->filter('.swiper-wrapper')
->first()
->filter('[data-background]')
->each(static function ($img) {
return $img->attr('data-background');
});
return $parsed_home;
}
} }

View File

@ -8,28 +8,27 @@
], ],
"license": "MIT", "license": "MIT",
"require": { "require": {
"php": "^8.0", "php": "^7.2.5",
"ext-json": "*", "ext-json": "*",
"artesaos/seotools": "^1.0",
"absmoca/leboncoin": "dev-master", "absmoca/leboncoin": "dev-master",
"artesaos/seotools": "^0.18.0",
"emanueleminotto/simple-html-dom": "^1.5", "emanueleminotto/simple-html-dom": "^1.5",
"fabpot/goutte": "^4.0", "fabpot/goutte": "^3.1",
"fideloper/proxy": "^4.2", "fideloper/proxy": "^4.2",
"fruitcake/laravel-cors": "^2.0", "fruitcake/laravel-cors": "^2.0",
"guzzlehttp/guzzle": "^7.0.1", "guzzlehttp/guzzle": "^6.3",
"laravel/framework": "^8.0", "laravel/framework": "^7.0",
"laravel/tinker": "^2.0", "laravel/tinker": "^2.0",
"laravel/ui": "^3.0", "laravel/ui": "^2.1",
"spatie/laravel-feed": "^4.0", "spatie/laravel-feed": "^2.7",
"spatie/laravel-query-builder": "^4.0", "spatie/laravel-query-builder": "^2.8"
"ext-intl": "*"
}, },
"require-dev": { "require-dev": {
"facade/ignition": "^2.3.6", "facade/ignition": "^2.0",
"fzaninotto/faker": "^1.9.1", "fzaninotto/faker": "^1.9.1",
"mockery/mockery": "^1.3.1", "mockery/mockery": "^1.3.1",
"nunomaduro/collision": "^5.0", "nunomaduro/collision": "^4.1",
"phpunit/phpunit": "^9.0" "phpunit/phpunit": "^8.5"
}, },
"config": { "config": {
"optimize-autoloader": true, "optimize-autoloader": true,

4969
composer.lock generated

File diff suppressed because it is too large Load Diff

View File

@ -22,18 +22,6 @@ return [
'description' => 'La sélection', 'description' => 'La sélection',
'language' => 'fr-FR', 'language' => 'fr-FR',
/*
* The image to display for the feed. For Atom feeds, this is displayed as
* a banner/logo; for RSS and JSON feeds, it's displayed as an icon.
* An empty value omits the image attribute from the feed.
*/
'image' => '',
/*
* The format of the feed. Acceptable values are 'rss', 'atom', or 'json'.
*/
'format' => 'atom',
/* /*
* The view that will render the feed. * The view that will render the feed.
*/ */
@ -43,12 +31,6 @@ return [
* The type to be used in the <link> tag * The type to be used in the <link> tag
*/ */
'type' => 'application/atom+xml', 'type' => 'application/atom+xml',
/*
* The content type for the feed response. Set to an empty string to automatically
* determine the correct value.
*/
'contentType' => '',
], ],
], ],
]; ];

View File

@ -0,0 +1,21 @@
<?php
namespace Tests\Feature;
use Illuminate\Foundation\Testing\RefreshDatabase;
use Tests\TestCase;
class ExampleTest extends TestCase
{
/**
* A basic test example.
*
* @return void
*/
public function testBasicTest()
{
$response = $this->get('/');
$response->assertStatus(200);
}
}

View File

@ -0,0 +1,18 @@
<?php
namespace Tests\Unit;
use PHPUnit\Framework\TestCase;
class ExampleTest extends TestCase
{
/**
* A basic test example.
*
* @return void
*/
public function testBasicTest()
{
$this->assertTrue(true);
}
}