* @copyright Copyright (c) 2018, Branko Kokanovic * * @author Branko Kokanovic * * @license GNU AGPL version 3 or any later version * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU Affero General Public License as * published by the Free Software Foundation, either version 3 of the * License, or (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU Affero General Public License for more details. * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . * */ namespace OCA\FaceRecognition\BackgroundJob\Tasks; use OCP\Image as OCP_Image; use OCP\Files\File; use OCP\Files\Folder; use OCP\Lock\ILockingProvider; use OCP\IUser; use OCA\FaceRecognition\BackgroundJob\FaceRecognitionBackgroundTask; use OCA\FaceRecognition\BackgroundJob\FaceRecognitionContext; use OCA\FaceRecognition\Db\Face; use OCA\FaceRecognition\Db\Image; use OCA\FaceRecognition\Db\ImageMapper; use OCA\FaceRecognition\Helper\TempImage; use OCA\FaceRecognition\Model\IModel; use OCA\FaceRecognition\Model\ModelManager; use OCA\FaceRecognition\Service\FileService; use OCA\FaceRecognition\Service\SettingsService; /** * Taks that get all images that are still not processed and processes them. * Processing image means that each image is prepared, faces extracted form it, * and for each found face - face descriptor is extracted. */ class ImageProcessingTask extends FaceRecognitionBackgroundTask { /** @var ImageMapper Image mapper*/ protected $imageMapper; /** @var FileService */ protected $fileService; /** @var SettingsService */ protected $settingsService; /** @var ModelManager $modelManager */ protected $modelManager; /** @var ILockingProvider $lockingProvider */ protected ILockingProvider $lockingProvider; /** @var IModel $model */ private $model; /** @var int|null $maxImageAreaCached Maximum image area (cached, so it is not recalculated for each image) */ private $maxImageAreaCached; /** * @param ImageMapper $imageMapper Image mapper * @param FileService $fileService * @param SettingsService $settingsService * @param ModelManager $modelManager Model manager * @param ILockingProvider $lockingProvider */ public function __construct(ImageMapper $imageMapper, FileService $fileService, SettingsService $settingsService, ModelManager $modelManager, ILockingProvider $lockingProvider) { parent::__construct(); $this->imageMapper = $imageMapper; $this->fileService = $fileService; $this->settingsService = $settingsService; $this->modelManager = $modelManager; $this->lockingProvider = $lockingProvider; $this->model = null; $this->maxImageAreaCached = null; } /** * @inheritdoc */ public function description() { return "Process all images to extract faces"; } /** * @inheritdoc */ public function execute(FaceRecognitionContext $context) { $this->setContext($context); $this->logInfo('NOTE: Starting face recognition. If you experience random crashes after this point, please look FAQ at https://github.com/matiasdelellis/facerecognition/wiki/FAQ'); // Get current model. $this->model = $this->modelManager->getCurrentModel(); // Open model. $this->model->open(); $images = $context->propertyBag['images']; foreach($images as $image) { yield; $startMillis = round(microtime(true) * 1000); try { // Get a image lock $lockKey = 'facerecognition/' . $image->getId(); $lockType = ILockingProvider::LOCK_EXCLUSIVE; $this->lockingProvider->acquireLock($lockKey, $lockType); $dbImage = $this->imageMapper->find($image->getUser(), $image->getId()); if ($dbImage->getIsProcessed()) { $this->logInfo('Faces found: 0. Image will be skipped since it was already processed.'); // Release lock of file. $this->lockingProvider->releaseLock($lockKey, $lockType); continue; } // Get an temp Image to process this image. $tempImage = $this->getTempImage($image); if (is_null($tempImage)) { // If we cannot find a file probably it was deleted out of our control and we must clean our tables. $this->settingsService->setNeedRemoveStaleImages(true, $image->user); $this->logInfo('File with ID ' . $image->file . ' doesn\'t exist anymore, skipping it'); // Release lock of file. $this->lockingProvider->releaseLock($lockKey, $lockType); continue; } if ($tempImage->getSkipped() === true) { $this->logInfo('Faces found: 0 (image will be skipped because it is too small)'); $this->imageMapper->imageProcessed($image, array(), 0); // Release lock of file. $this->lockingProvider->releaseLock($lockKey, $lockType); continue; } // Get faces in the temporary image $tempImagePath = $tempImage->getTempPath(); $rawFaces = $this->model->detectFaces($tempImagePath); $this->logInfo('Faces found: ' . count($rawFaces)); $faces = array(); foreach ($rawFaces as $rawFace) { // Normalize face and landmarks from model to original size $normFace = $this->getNormalizedFace($rawFace, $tempImage->getRatio()); // Convert from dictionary of face to our Face Db Entity. $face = Face::fromModel($image->getId(), $normFace); // Save the normalized Face to insert on database later. $faces[] = $face; } // Save new faces fo database $endMillis = round(microtime(true) * 1000); $duration = (int) max($endMillis - $startMillis, 0); $this->imageMapper->imageProcessed($image, $faces, $duration); // Release lock of file. $this->lockingProvider->releaseLock($lockKey, $lockType); } catch (\OCP\Lock\LockedException $e) { $this->logInfo('Faces found: 0. Image will be skipped because it is locked'); } catch (\Exception $e) { if ($e->getMessage() === "std::bad_alloc") { throw new \RuntimeException("Not enough memory to run face recognition! Please look FAQ at https://github.com/matiasdelellis/facerecognition/wiki/FAQ"); } $this->logInfo('Faces found: 0. Image will be skipped because of the following error: ' . $e->getMessage()); $this->logDebug((string) $e); // Save an empty entry so it can be analyzed again later $this->imageMapper->imageProcessed($image, array(), 0, $e); } finally { // Clean temporary image. if (isset($tempImage)) { $tempImage->clean(); } // If there are temporary files from external files, they must also be cleaned. $this->fileService->clean(); } } return true; } /** * Given an image, build a temporary image to perform the analysis * * return TempImage|null */ private function getTempImage(Image $image): ?TempImage { // todo: check if this hits I/O (database, disk...), consider having lazy caching to return user folder from user $file = $this->fileService->getFileById($image->getFile(), $image->getUser()); if (empty($file)) { return null; } if (!$this->fileService->isAllowedNode($file)) { return null; } $imagePath = $this->fileService->getLocalFile($file); if ($imagePath === null) return null; $this->logInfo('Processing image ' . $imagePath); $tempImage = new TempImage($imagePath, $this->model->getPreferredMimeType(), $this->getMaxImageArea(), $this->settingsService->getMinimumImageSize()); return $tempImage; } /** * Obtains max image area lazily (from cache, or calculates it and puts it to cache) * * @return int Max image area (in pixels^2) */ private function getMaxImageArea(): int { // First check if is cached // if (!is_null($this->maxImageAreaCached)) { return $this->maxImageAreaCached; } // Get this setting on main app_config. // Note that this option has lower and upper limits and validations $this->maxImageAreaCached = $this->settingsService->getAnalysisImageArea(); // Check if admin override it in config and it is valid value // $maxImageArea = $this->settingsService->getMaximumImageArea(); if ($maxImageArea > 0) { $this->maxImageAreaCached = $maxImageArea; } // Also check if we are provided value from command line. // if ((array_key_exists('max_image_area', $this->context->propertyBag)) && (!is_null($this->context->propertyBag['max_image_area']))) { $this->maxImageAreaCached = $this->context->propertyBag['max_image_area']; } return $this->maxImageAreaCached; } /** * Helper method, to normalize face sizes back to original dimensions, based on ratio * */ private function getNormalizedFace(array $rawFace, float $ratio): array { $face = []; $face['left'] = intval(round($rawFace['left']*$ratio)); $face['right'] = intval(round($rawFace['right']*$ratio)); $face['top'] = intval(round($rawFace['top']*$ratio)); $face['bottom'] = intval(round($rawFace['bottom']*$ratio)); $face['detection_confidence'] = $rawFace['detection_confidence']; $face['landmarks'] = $this->getNormalizedLandmarks($rawFace['landmarks'], $ratio); $face['descriptor'] = $rawFace['descriptor']; return $face; } /** * Helper method, to normalize landmarks sizes back to original dimensions, based on ratio * */ private function getNormalizedLandmarks(array $rawLandmarks, float $ratio): array { $landmarks = []; foreach ($rawLandmarks as $rawLandmark) { $landmark = []; $landmark['x'] = intval(round($rawLandmark['x']*$ratio)); $landmark['y'] = intval(round($rawLandmark['y']*$ratio)); $landmarks[] = $landmark; } return $landmarks; } }