bookmarkMapper = $bookmarkMapper; $this->bookmarkPreviewer = $bookmarkPreviewer; $this->faviconPreviewer = $faviconPreviewer; $this->config = $config; $this->rootFolder = $rootFolder; $this->l = $l; $this->mimey = new MimeTypes; $this->logger = $logger; } /** * @param Bookmark $bookmark * @throws UrlParseError */ public function crawl(Bookmark $bookmark): void { try { $client = new Client(['connect_timeout' => self::CONNECT_TIMEOUT, 'read_timeout' => self::READ_TIMEOUT]); /** @var Response $resp */ $resp = $client->get($bookmark->getUrl()); $available = $resp ? $resp->getStatusCode() !== 404 : false; } catch (Exception $e) { $available = false; } if ($available) { $this->archiveFile($bookmark, $resp); $this->archiveContent($bookmark, $resp); $this->bookmarkPreviewer->getImage($bookmark); $this->faviconPreviewer->getImage($bookmark); } $bookmark->markPreviewCreated(); $bookmark->setAvailable($available); $this->bookmarkMapper->update($bookmark); } private function archiveContent(Bookmark $bookmark, Response $resp) : void { $contentType = $resp->getHeader('Content-type')[0]; if ((bool)preg_match('#text/html#i', $contentType) === true && ($bookmark->getHtmlContent() === null || $bookmark->getHtmlContent() === '')) { $config = new Configuration(); $config ->setFixRelativeURLs(true) ->setOriginalURL($bookmark->getUrl()) ->setSubstituteEntities(true); $readability = new Readability($config); try { $readability->parse($resp->getBody()); } catch (ParseException $e) { $this->logger->debug(get_class($e)." ".$e->getMessage()."\r\n".$e->getTraceAsString()); } $bookmark->setHtmlContent($readability->getContent()); $bookmark->setTextContent(strip_tags($readability->getContent())); } } private function archiveFile(Bookmark $bookmark, Response $resp) :void { $contentType = $resp->getHeader('Content-type')[0]; if ((bool)preg_match('#text/html#i', $contentType) === false && $bookmark->getArchivedFile() === null && (int)$resp->getHeader('Content-length')[0] < self::MAX_BODY_LENGTH) { try { $userFolder = $this->rootFolder->getUserFolder($bookmark->getUserId()); $folderPath = $this->getArchivePath($bookmark, $userFolder); $name = $bookmark->slugify('title'); $path = $folderPath . '/' . $name . '.' . $this->mimey->getExtension($contentType); $i = 0; while ($userFolder->nodeExists($path)) { $path = $folderPath . '/' .$name . '_' . $i . '.' . $this->mimey->getExtension($contentType); $i++; } $file = $userFolder->newFile($path); $file->putContent($resp->getBody()); $bookmark->setArchivedFile($file->getId()); $this->bookmarkMapper->update($bookmark); } catch (NotPermittedException | NoUserException | GenericFileException | LockedException | UrlParseError | InvalidPathException | NotFoundException $e) { $this->logger->debug(get_class($e)." ".$e->getMessage()."\r\n".$e->getTraceAsString()); } } } private function getArchivePath(Bookmark $bookmark, Folder $userFolder): string { $folderPath = $this->config->getUserValue($bookmark->getUserId(), 'bookmarks', 'archive.filePath', $this->l->t('Bookmarks')); $this->getOrCreateFolder($userFolder, $folderPath); return $folderPath; } public function getOrCreateFolder(Folder $userFolder, string $path) : ?Folder { if ($path === '/') { return $userFolder; } if ($userFolder->nodeExists($path)) { $folder = $userFolder->get($path); } else { $folder = $userFolder->newFolder($path); } if (!($folder instanceof Folder)) { return null; } return $folder; } }