ReuserAgent.php

Go to the documentation of this file.

<?php
/*
 SPDX-FileCopyrightText: © 2014-2018 Siemens AG
Author: Daniele Fognini, Andreas Würl
 
 SPDX-License-Identifier: GPL-2.0-only
*/
 
namespace Fossology\Reuser;
 
use Fossology\Lib\Agent\Agent;
use Fossology\Lib\BusinessRules\AgentLicenseEventProcessor;
use Fossology\Lib\BusinessRules\ClearingDecisionFilter;
use Fossology\Lib\BusinessRules\ClearingDecisionProcessor;
use Fossology\Lib\BusinessRules\ClearingEventProcessor;
use Fossology\Lib\Dao\ClearingDao;
use Fossology\Lib\Dao\UploadDao;
use Fossology\Lib\Dao\AgentDao;
use Fossology\Lib\Proxy\ScanJobProxy;
use Fossology\Lib\Dao\CopyrightDao;
use Fossology\Lib\Data\ClearingDecision;
use Fossology\Lib\Data\DecisionTypes;
use Fossology\Lib\Data\Tree\Item;
use Fossology\Lib\Util\ArrayOperation;
use Fossology\Lib\Data\Clearing\ClearingEventTypes;
use Fossology\Lib\Data\Tree\ItemTreeBounds;
 
include_once(__DIR__ . "/version.php");
 
class ReuserAgent extends Agent
{
  private $uploadDao;
  private $clearingEventProcessor;
  private $agentLicenseEventProcessor;
  private $clearingDecisionFilter;
  private $clearingDecisionProcessor;
  private $clearingDao;
  private $copyrightDao;
  private $aagentDao;
  private $decisionTypes;
 
  function __construct()
  {
    parent::__construct(REUSER_AGENT_NAME, AGENT_VERSION, AGENT_REV);
    $this->aagentDao = $this->container->get('dao.agent');
    $this->uploadDao = $this->container->get('dao.upload');
    $this->clearingDao = $this->container->get('dao.clearing');
    $this->copyrightDao = $this->container->get('dao.copyright');
    $this->decisionTypes = $this->container->get('decision.types');
    $this->clearingEventProcessor = $this->container->get('businessrules.clearing_event_processor');
    $this->clearingDecisionFilter = $this->container->get('businessrules.clearing_decision_filter');
    $this->clearingDecisionProcessor = $this->container->get('businessrules.clearing_decision_processor');
    $this->agentLicenseEventProcessor = $this->container->get('businessrules.agent_license_event_processor');
  }
 
  function processUploadId($uploadId)
  {
    $itemTreeBounds = $this->uploadDao->getParentItemBounds($uploadId);
    foreach ($this->uploadDao->getReusedUpload($uploadId, $this->groupId) as $reuseTriple) {
      // Get the reuse upload id
      $reusedUploadId = $reuseTriple['reused_upload_fk'];
      // Get the group id
      $reusedGroupId = $reuseTriple['reused_group_fk'];
      // Get the reuse mode
      $reuseMode = $reuseTriple['reuse_mode'];
      // Get the ItemTreeBounds for the upload
      $itemTreeBoundsReused = $this->uploadDao->getParentItemBounds($reusedUploadId);
      if (false === $itemTreeBoundsReused) {
        continue;
      }
 
      if ($reuseMode & UploadDao::REUSE_ENHANCED) {
        $this->processEnhancedUploadReuse($itemTreeBounds, $itemTreeBoundsReused, $reusedGroupId);
      } else {
        $this->processUploadReuse($itemTreeBounds, $itemTreeBoundsReused, $reusedGroupId);
      }
 
      if ($reuseMode & UploadDao::REUSE_MAIN) {
        $this->reuseMainLicense($uploadId, $this->groupId, $reusedUploadId, $reusedGroupId);
      }
 
      if ($reuseMode & UploadDao::REUSE_CONF) {
        $this->reuseConfSettings($uploadId, $reusedUploadId);
      }
 
      if ($reuseMode & UploadDao::REUSE_COPYRIGHT) {
        $this->reuseCopyrights($uploadId, $reusedUploadId);
      }
    }
    return true;
  }
  protected function reuseConfSettings($uploadId, $reusedUploadId)
  {
    if (!$this->uploadDao->insertReportConfReuse($uploadId, $reusedUploadId)) {
      echo "INFO :: Report configuration for select upload doesn't exists. Unable to copy!!!";
    }
    $this->heartbeat(1);
    return true;
  }
 
  protected function reuseMainLicense($uploadId, $groupId, $reusedUploadId, $reusedGroupId)
  {
    $mainLicenseIds = $this->clearingDao->getMainLicenseIds($reusedUploadId, $reusedGroupId);
    if (!empty($mainLicenseIds)) {
      $existingMainLicenseIds = $this->clearingDao->getMainLicenseIds($uploadId, $groupId);
      foreach ($mainLicenseIds as $mainLicenseId) {
        if (in_array($mainLicenseId, $existingMainLicenseIds)) {
          continue;
        } else {
          $this->clearingDao->makeMainLicense($uploadId, $groupId, $mainLicenseId);
        }
      }
    }
    return true;
  }
 
  protected function getAgentId($uploadId)
  {
    $agentName = 'copyright';
 
    $scanJobProxy = new ScanJobProxy($this->aagentDao, $uploadId);
 
    $scanJobProxy->createAgentStatus(array($agentName));
    $selectedScanners = $scanJobProxy->getLatestSuccessfulAgentIds();
    if (!array_key_exists($agentName, $selectedScanners)) {
      return;
    }
    return $selectedScanners[$agentName];
  }
 
 
  protected function reuseCopyrights($uploadId, $reusedUploadId)
  {
    $agentId = $this->getAgentId($uploadId);
    $reusedAgentId = $this->getAgentId($reusedUploadId);
    if ($agentId == null || $reusedAgentId == null) {
      return true;
    }
    $uploadTreeTableName = $this->uploadDao->getUploadtreeTableName($uploadId);
    $extrawhere = ' agent_fk='.$agentId;
    $allCopyrights = $this->copyrightDao->getScannerEntries('copyright',
      $uploadTreeTableName, $uploadId, null, $extrawhere);
 
    $reusedCopyrights = $this->copyrightDao->getAllEventEntriesForUpload(
      $reusedUploadId, $reusedAgentId);
 
    if (!empty($reusedCopyrights) && !empty($allCopyrights)) {
      // Pre-index $allCopyrights by hash to avoid nested loop O(N*M)
      $copyrightsByHash = [];
      foreach ($allCopyrights as $copyrightKey => $copyright) {
        $hash = $copyright['hash'];
        if (!isset($copyrightsByHash[$hash])) {
          $copyrightsByHash[$hash] = [];
        }
        $copyrightsByHash[$hash][] = $copyrightKey;
      }
 
      foreach ($reusedCopyrights as $reusedCopyright) {
        $reusedHash = $reusedCopyright['hash'];
        if (isset($copyrightsByHash[$reusedHash]) && !empty($copyrightsByHash[$reusedHash])) {
          // Take the first matching entry
          $copyrightKey = array_shift($copyrightsByHash[$reusedHash]);
          $copyright = $allCopyrights[$copyrightKey];
 
          if ($this->dbManager->booleanFromDb($reusedCopyright['is_enabled'])) {
            $action = "update";
            $content = $reusedCopyright["contentedited"];
          } else {
            $action = "delete";
            $content = "";
          }
          $hash = $copyright['hash'];
          $item = new ItemTreeBounds(
                    intval($copyright['uploadtree_pk']), $uploadTreeTableName,
                    $copyright['upload_fk'], $copyright['lft'], $copyright['rgt']);
          $this->copyrightDao->updateTable($item, $hash, $content,
            $this->userId, 'copyright', $action);
          $this->heartbeat(1);
        }
      }
    }
    return true;
  }
 
  protected function processUploadReuse($itemTreeBounds, $itemTreeBoundsReused, $reusedGroupId)
  {
    $groupId = $this->groupId;
    $userId = $this->userId;
 
    $clearingDecisions = $this->clearingDao->getFileClearingsFolder($itemTreeBoundsReused, $reusedGroupId);
    $currenlyVisibleClearingDecisions = $this->clearingDao->getFileClearingsFolder($itemTreeBounds, $groupId);
 
    $currenlyVisibleClearingDecisionsById = $this->mapByClearingId($currenlyVisibleClearingDecisions);
    $clearingDecisionsById = $this->mapByClearingId($clearingDecisions);
 
    $clearingDecisionsToImport = array_diff_key($clearingDecisionsById,$currenlyVisibleClearingDecisionsById);
 
    $clearingDecisionToImportByFileId = $this->mapByFileId($clearingDecisionsToImport);
 
    $uploadDao = $this->uploadDao;
    $containedItems = ArrayOperation::callChunked(
        function ($fileIds) use ($itemTreeBounds, $uploadDao)
        {
          return $uploadDao->getContainedItems(
              $itemTreeBounds,
              "pfile_fk = ANY($1)",
              array('{' . implode(', ', $fileIds) . '}')
          );
        }, array_keys($clearingDecisionToImportByFileId), 100);
 
    foreach ($containedItems as $item) {
      $fileId = $item->getFileId();
      if (empty($fileId)) {
        continue;
      }
      if (array_key_exists($fileId, $clearingDecisionToImportByFileId)) {
        $this->createCopyOfClearingDecision($item->getId(), $userId, $groupId,
          $clearingDecisionToImportByFileId[$fileId]);
      } else {
        throw new \Exception("bad internal state");
      }
 
      $this->heartbeat(1);
    }
 
    return true;
  }
 
  protected function processEnhancedUploadReuse($itemTreeBounds, $itemTreeBoundsReused, $reusedGroupId)
  {
    $clearingDecisions = $this->clearingDao->getFileClearingsFolder($itemTreeBoundsReused, $reusedGroupId);
    $currenlyVisibleClearingDecisions = $this->clearingDao->getFileClearingsFolder($itemTreeBounds, $this->groupId);
 
    $currenlyVisibleClearingDecisionsById = $this->mapByClearingId($currenlyVisibleClearingDecisions);
    $clearingDecisionsById = $this->mapByClearingId($clearingDecisions);
 
    $clearingDecisionsToImport = array_diff_key($clearingDecisionsById,$currenlyVisibleClearingDecisionsById);
 
    $sql = "SELECT ut.* FROM uploadtree ur, uploadtree ut WHERE ur.upload_fk=$2"
         . " AND ur.pfile_fk=$3 AND ut.upload_fk=$1 AND ut.ufile_name=ur.ufile_name";
    $stmt = __METHOD__.'.reuseByName';
    $this->dbManager->prepare($stmt, $sql);
    $treeDao = $this->container->get('dao.tree');
 
    foreach ($clearingDecisionsToImport as $clearingDecision) {
      $reusedPath = $treeDao->getRepoPathOfPfile($clearingDecision->getPfileId());
      if (empty($reusedPath)) {
        // File missing from repo
        continue;
      }
 
      $res = $this->dbManager->execute($stmt,array($itemTreeBounds->getUploadId(),
        $itemTreeBoundsReused->getUploadId(),$clearingDecision->getPfileId()));
      while ($row = $this->dbManager->fetchArray($res)) {
        $newPath = $treeDao->getRepoPathOfPfile($row['pfile_fk']);
        if (empty($newPath)) {
          // File missing from repo
          continue;
        }
        $this->copyClearingDecisionIfDifferenceIsSmall($reusedPath, $newPath, $clearingDecision, $row['uploadtree_pk']);
      }
      $this->dbManager->freeResult($res);
    }
  }
 
  protected function copyClearingDecisionIfDifferenceIsSmall($reusedPath,$newPath,$clearingDecision,$itemId)
  {
    $diffLevel = exec("diff $reusedPath $newPath | wc -l");
    if ($diffLevel === false) {
      throw new \Exception('cannot use diff tool');
    }
    if ($diffLevel < 5) {
      $this->createCopyOfClearingDecision($itemId, $this->userId, $this->groupId, $clearingDecision);
      $this->heartbeat(1);
    }
  }
 
  protected function mapByFileId($clearingDecisions)
  {
    $clearingDecisionByFileId = array();
    foreach ($clearingDecisions as $clearingDecision) {
      $fileId = $clearingDecision->getPfileId();
      if (!array_key_exists($fileId, $clearingDecisionByFileId)) {
        $clearingDecisionByFileId[$fileId] = $clearingDecision;
      }
    }
    return $clearingDecisionByFileId;
  }
 
  protected function createCopyOfClearingDecision($itemId, $userId, $groupId, $clearingDecisionToCopy)
  {
    $clearingEventIdsToCopy = array();
    foreach ($clearingDecisionToCopy->getClearingEvents() as $clearingEvent) {
      $licenseId = $clearingEvent->getLicenseId();
      $uploadTreeId = $itemId;
      $isRemoved = $clearingEvent->isRemoved();
      $type = ClearingEventTypes::USER;
      $reportInfo = $clearingEvent->getReportinfo();
      $comment = $clearingEvent->getComment();
      $acknowledgement = $clearingEvent->getAcknowledgement();
      $jobId = $this->jobId;
      $clearingEventIdsToCopy[] = $this->clearingDao->insertClearingEvent(
        $uploadTreeId, $userId, $groupId, $licenseId, $isRemoved,
        $type, $reportInfo, $comment, $acknowledgement, $jobId);
    }
 
    $this->clearingDao->createDecisionFromEvents(
        $itemId,
        $userId,
        $groupId,
        $clearingDecisionToCopy->getType(),
        $clearingDecisionToCopy->getScope(),
        $clearingEventIdsToCopy
    );
  }
 
  public function mapByClearingId($clearingDecisions)
  {
    $mapped = array();
 
    foreach ($clearingDecisions as $clearingDecision) {
      $mapped[$clearingDecision->getClearingId()] = $clearingDecision;
    }
 
    return $mapped;
  }
}