DeciderAgent.php

Go to the documentation of this file.

<?php
/*
 Author: Daniele Fognini
 SPDX-FileCopyrightText: © 2014-2019 Siemens AG
 SPDX-FileCopyrightText: © 2021 Orange by Piotr Pszczola <piotr.pszczola@orange.com>
 SPDX-FileCopyrightText: © 2021 Kaushlendra Pratap <kaushlendrapratap.9837@gmail.com>
 SPDX-FileCopyrightText: © Fossology contributors
 
 SPDX-License-Identifier: GPL-2.0-only
*/
namespace Fossology\Decider;
 
use Fossology\Lib\Agent\Agent;
use Fossology\Lib\BusinessRules\AgentLicenseEventProcessor;
use Fossology\Lib\BusinessRules\ClearingDecisionProcessor;
use Fossology\Lib\BusinessRules\LicenseMap;
use Fossology\Lib\Dao\ClearingDao;
use Fossology\Lib\Dao\CompatibilityDao;
use Fossology\Lib\Dao\CopyrightDao;
use Fossology\Lib\Dao\HighlightDao;
use Fossology\Lib\Dao\LicenseDao;
use Fossology\Lib\Dao\ShowJobsDao;
use Fossology\Lib\Dao\UploadDao;
use Fossology\Lib\Data\DecisionScopes;
use Fossology\Lib\Data\DecisionTypes;
use Fossology\Lib\Data\Clearing\ClearingEventTypes;
use Fossology\Lib\Data\LicenseMatch;
use Fossology\Lib\Data\Tree\Item;
use Fossology\Lib\Data\Tree\ItemTreeBounds;
use Fossology\Lib\Exceptions\InvalidAgentStageException;
use Fossology\Lib\Proxy\ScanJobProxy;
use Symfony\Component\Process\Process;
 
include_once(__DIR__ . "/version.php");
 
class DeciderAgent extends Agent
{
  const RULES_NOMOS_IN_MONK = 0x1;
  const RULES_NOMOS_MONK_NINKA = 0x2;
  const RULES_BULK_REUSE = 0x4;
  const RULES_WIP_SCANNER_UPDATES = 0x8;
  const RULES_OJO_NO_CONTRADICTION = 0x10;
  const RULES_COPYRIGHT_FALSE_POSITIVE = 0x20;
  const RULES_COPYRIGHT_FALSE_POSITIVE_CLUTTER = 0x40;
  const RULES_LICENSE_TYPE_CONCLUSION = 0x80;
  const RULES_KOTOBA_NO_CONTRADICTION = 0x100;
  const RULES_RESO_NO_CONTRADICTION = 0x200;
  const RULES_ALL = self::RULES_NOMOS_IN_MONK | self::RULES_NOMOS_MONK_NINKA |
    self::RULES_BULK_REUSE | self::RULES_WIP_SCANNER_UPDATES |
    self::RULES_OJO_NO_CONTRADICTION | self::RULES_RESO_NO_CONTRADICTION |
    self::RULES_LICENSE_TYPE_CONCLUSION | self::RULES_KOTOBA_NO_CONTRADICTION;
 
  private $activeRules;
  private $licenseType;
  private $uploadDao;
  private $clearingDecisionProcessor;
  private $agentLicenseEventProcessor;
  private $clearingDao;
  private $highlightDao;
  private $showJobsDao;
  private $decisionTypes;
  private $licenseMap = null;
  private $licenseMapUsage = null;
 
  private $copyrightDao;
 
  private $compatibilityDao;
 
  private $licenseDao;
 
  function __construct($licenseMapUsage=null)
  {
    parent::__construct(AGENT_DECIDER_NAME, AGENT_DECIDER_VERSION, AGENT_DECIDER_REV);
 
    $this->uploadDao = $this->container->get('dao.upload');
    $this->clearingDao = $this->container->get('dao.clearing');
    $this->highlightDao = $this->container->get('dao.highlight');
    $this->showJobsDao = $this->container->get('dao.show_jobs');
    $this->decisionTypes = $this->container->get('decision.types');
    $this->clearingDecisionProcessor = $this->container->get('businessrules.clearing_decision_processor');
    $this->agentLicenseEventProcessor = $this->container->get('businessrules.agent_license_event_processor');
    $this->copyrightDao = $this->container->get('dao.copyright');
    $this->compatibilityDao = $this->container->get('dao.compatibility');
    $this->licenseDao = $this->container->get('dao.license');
    $this->licenseMapUsage = $licenseMapUsage;
    $this->agentSpecifOptions = "r:t:";
  }
 
  function processUploadId($uploadId)
  {
    $args = $this->args;
    $this->activeRules = array_key_exists('r', $args) ? intval($args['r']) : self::RULES_ALL;
    $this->licenseType = array_key_exists('t', $args) ?
        $this->getLicenseType(str_replace(["'", '"'], "", $args['t'])) : "";
    $this->licenseMap = new LicenseMap($this->dbManager, $this->groupId, $this->licenseMapUsage);
 
    if (array_key_exists("r", $args) && (($this->activeRules&self::RULES_COPYRIGHT_FALSE_POSITIVE)== self::RULES_COPYRIGHT_FALSE_POSITIVE)) {
      $this->getCopyrightsToDisableFalsePositivesClutter($uploadId, false);
    }
    if (array_key_exists("r", $args) && (($this->activeRules&self::RULES_COPYRIGHT_FALSE_POSITIVE_CLUTTER)== self::RULES_COPYRIGHT_FALSE_POSITIVE_CLUTTER)) {
      $this->getCopyrightsToDisableFalsePositivesClutter($uploadId, true);
    }
    if (array_key_exists("r", $args) && (($this->activeRules&self::RULES_BULK_REUSE)== self::RULES_BULK_REUSE)) {
      $bulkReuser = new BulkReuser();
      $bulkIds = $this->clearingDao->getPreviousBulkIds($uploadId, $this->groupId, $this->userId);
      if (count($bulkIds) == 0) {
        return true;
      }
      $jqId=0;
      $minTime="4";
      $maxTime="60";
      foreach ($bulkIds as $bulkId) {
        $jqId = $bulkReuser->rerunBulkAndDeciderOnUpload($uploadId, $this->groupId, $this->userId, $bulkId, $jqId);
        $this->heartbeat(1);
        if (!empty($jqId)) {
          $jqIdRow = $this->showJobsDao->getDataForASingleJob($jqId);
          while ($this->showJobsDao->getJobStatus($jqId)) {
            $this->heartbeat(0);
            $timeInSec = $this->showJobsDao->getEstimatedTime($jqIdRow['jq_job_fk'],'',0,0,1);
            if ($timeInSec > $maxTime) {
              sleep($maxTime);
            } else if ($timeInSec < $minTime) {
              sleep($minTime);
            } else {
              sleep($timeInSec);
            }
          }
        }
      }
    }
    $parentBounds = $this->uploadDao->getParentItemBounds($uploadId);
    foreach ($this->uploadDao->getContainedItems($parentBounds) as $item) {
      $process = $this->processItem($item);
      $this->heartbeat($process);
    }
    return true;
  }
 
  private function processItem(Item $item)
  {
    $itemTreeBounds = $item->getItemTreeBounds();
 
    $unMappedMatches = $this->agentLicenseEventProcessor->getLatestScannerDetectedMatches($itemTreeBounds);
    $projectedScannerMatches = $this->remapByProjectedId($unMappedMatches);
 
    $lastDecision = $this->clearingDao->getRelevantClearingDecision($itemTreeBounds, $this->groupId);
 
    if (null!==$lastDecision && $lastDecision->getType()==DecisionTypes::IRRELEVANT) {
      return 0;
    }
 
    $currentEvents = $this->clearingDao->getRelevantClearingEvents($itemTreeBounds, $this->groupId);
 
    $markAsWip = false;
    if (null !== $lastDecision && $projectedScannerMatches
      && ($this->activeRules & self::RULES_WIP_SCANNER_UPDATES) == self::RULES_WIP_SCANNER_UPDATES) {
      $licensesFromDecision = array();
      foreach ($lastDecision->getClearingLicenses() as $clearingLicense) {
        $licenseIdFromEvent = $this->licenseMap->getProjectedId($clearingLicense->getLicenseId());
        $licensesFromDecision[$licenseIdFromEvent] = $licenseIdFromEvent;
      }
      $markAsWip = $this->existsUnhandledMatch($projectedScannerMatches,$licensesFromDecision);
    }
 
    if (null !== $lastDecision && $markAsWip) {
      $this->clearingDao->markDecisionAsWip($item->getId(), $this->userId, $this->groupId);
      return 1;
    }
 
    /* Kotoba stores findings as clearing events, which would trip the guard
       below. Auto-conclude here when all events are from kotoba and no scanner
       license is unaccounted for. */
    if (null === $lastDecision && count($currentEvents) > 0
        && ($this->activeRules & self::RULES_KOTOBA_NO_CONTRADICTION) == self::RULES_KOTOBA_NO_CONTRADICTION) {
      if ($this->autodecideIfKotobaMatchesNoContradiction($itemTreeBounds, $projectedScannerMatches, $currentEvents)) {
        return 1;
      }
    }
 
    if (null!==$lastDecision || 0<count($currentEvents)) {
      return 0;
    }
 
    $haveDecided = false;
 
    if (!$haveDecided && ($this->activeRules&self::RULES_OJO_NO_CONTRADICTION) == self::RULES_OJO_NO_CONTRADICTION) {
      $haveDecided = $this->autodecideIfOjoMatchesNoContradiction($itemTreeBounds, $projectedScannerMatches);
    }
 
    if (!$haveDecided && ($this->activeRules&self::RULES_RESO_NO_CONTRADICTION) == self::RULES_RESO_NO_CONTRADICTION) {
      $haveDecided = $this->autodecideIfResoMatchesNoContradiction($itemTreeBounds, $projectedScannerMatches);
    }
 
    if (!$haveDecided && ($this->activeRules&self::RULES_NOMOS_IN_MONK) == self::RULES_NOMOS_IN_MONK) {
      $haveDecided = $this->autodecideNomosMatchesInsideMonk($itemTreeBounds, $projectedScannerMatches);
    }
 
    if (!$haveDecided && ($this->activeRules&self::RULES_NOMOS_MONK_NINKA)== self::RULES_NOMOS_MONK_NINKA) {
      $haveDecided = $this->autodecideNomosMonkNinka($itemTreeBounds, $projectedScannerMatches);
    }
 
    if (!$haveDecided && ($this->activeRules &
            self::RULES_LICENSE_TYPE_CONCLUSION) ==
        self::RULES_LICENSE_TYPE_CONCLUSION) {
      $haveDecided = $this->autodecideLicenseType($itemTreeBounds,
          $projectedScannerMatches);
    }
 
    if (!$haveDecided && $markAsWip) {
      $this->clearingDao->markDecisionAsWip($item->getId(), $this->userId, $this->groupId);
    }
 
    return ($haveDecided||$markAsWip ? 1 : 0);
  }
 
  private function existsUnhandledMatch($projectedScannerMatches, $licensesFromDecision)
  {
    foreach (array_keys($projectedScannerMatches) as $projectedLicenseId) {
      if (!array_key_exists($projectedLicenseId, $licensesFromDecision)) {
        return true;
      }
    }
    return false;
  }
 
  private function autodecideIfOjoMatchesNoContradiction(ItemTreeBounds $itemTreeBounds, $matches)
  {
    $licenseMatchExists = count($matches) > 0;
    foreach ($matches as $licenseMatches) {
      $licenseMatchExists = $licenseMatchExists && $this->areOtherScannerFindingsAndOJOAgreed($licenseMatches);
    }
 
    if ($licenseMatchExists) {
      try {
        $this->clearingDecisionProcessor->makeDecisionFromLastEvents(
          $itemTreeBounds, $this->userId, $this->groupId,
          DecisionTypes::IDENTIFIED, false);
      } catch (\Exception $e) {
        echo "Can not auto decide as file '" .
          $itemTreeBounds->getItemId() . "' contains candidate license.\n";
      }
    }
    return $licenseMatchExists;
  }
 
  private function autodecideIfKotobaMatchesNoContradiction(ItemTreeBounds $itemTreeBounds, $projectedScannerMatches, $currentEvents)
  {
    $accountedLicenses = array();
    foreach ($currentEvents as $event) {
      if ($event->getEventType() != ClearingEventTypes::KOTOBA) {
        return false; // human event present; do not override
      }
      $projectedId = $this->licenseMap->getProjectedId($event->getLicenseId());
      $accountedLicenses[$projectedId] = true;
    }
 
    foreach (array_keys($projectedScannerMatches) as $projectedLicenseId) {
      if (!array_key_exists($projectedLicenseId, $accountedLicenses)) {
        return false; // unaccounted scanner license: contradiction
      }
    }
 
    try {
      $this->clearingDecisionProcessor->makeDecisionFromLastEvents(
        $itemTreeBounds, $this->userId, $this->groupId,
        DecisionTypes::IDENTIFIED, false);
    } catch (\Exception $e) {
      echo "Can not auto decide as file '" .
        $itemTreeBounds->getItemId() . "' contains candidate license.\n";
      return false;
    }
    return true;
  }
 
  private function autodecideIfResoMatchesNoContradiction(ItemTreeBounds $itemTreeBounds, $matches)
  {
    $licenseMatchExists = count($matches) > 0;
    foreach ($matches as $licenseMatches) {
      $licenseMatchExists = $licenseMatchExists && $this->areOtherScannerFindingsAndRESOAgreed($licenseMatches);
    }
 
    if ($licenseMatchExists) {
      try {
        $this->clearingDecisionProcessor->makeDecisionFromLastEvents(
          $itemTreeBounds, $this->userId, $this->groupId,
          DecisionTypes::IDENTIFIED, false);
      } catch (\Exception $e) {
        echo "Can not auto decide as file '" .
          $itemTreeBounds->getItemId() . "' contains candidate license.\n";
      }
    }
    return $licenseMatchExists;
  }
 
  private function autodecideNomosMonkNinka(ItemTreeBounds $itemTreeBounds, $matches)
  {
    $canDecide = (count($matches)>0);
 
    foreach ($matches as $licenseMatches) {
      if (!$canDecide) { // &= is not lazy
        break;
      }
      $canDecide &= $this->areNomosMonkNinkaAgreed($licenseMatches);
    }
 
    if ($canDecide) {
      $this->clearingDecisionProcessor->makeDecisionFromLastEvents($itemTreeBounds, $this->userId, $this->groupId, DecisionTypes::IDENTIFIED, $global=true);
    }
    return $canDecide;
  }
 
  private function autodecideNomosMatchesInsideMonk(ItemTreeBounds $itemTreeBounds, $matches)
  {
    $canDecide = (count($matches)>0);
 
    foreach ($matches as $licenseMatches) {
      if (!$canDecide) { // &= is not lazy
        break;
      }
      $canDecide &= $this->areNomosMatchesInsideAMonkMatch($licenseMatches);
    }
 
    if ($canDecide) {
      $this->clearingDecisionProcessor->makeDecisionFromLastEvents($itemTreeBounds, $this->userId, $this->groupId, DecisionTypes::IDENTIFIED, $global=true);
    }
    return $canDecide;
  }
 
  private function autodecideLicenseType(ItemTreeBounds $itemTreeBounds,
                                         $matches)
  {
    $canDecide = $this->noLicenseConflict($itemTreeBounds, $matches);
    if ($canDecide) {
      $canDecide &= $this->allLicenseInType($matches);
    }
 
    if ($canDecide) {
      $this->clearingDecisionProcessor
          ->makeDecisionFromLastEvents($itemTreeBounds, $this->userId,
              $this->groupId, DecisionTypes::IDENTIFIED, DecisionScopes::ITEM, [], true);
    }
    return $canDecide;
  }
 
  protected function remapByProjectedId($matches)
  {
    $remapped = array();
    foreach ($matches as $licenseId => $licenseMatches) {
      $projectedId = $this->licenseMap->getProjectedId($licenseId);
 
      foreach ($licenseMatches as $agent => $agentMatches) {
        $haveId = array_key_exists($projectedId, $remapped);
        $haveAgent = $haveId && array_key_exists($agent, $remapped[$projectedId]);
        if ($haveAgent) {
          $remapped[$projectedId][$agent] = array_merge($remapped[$projectedId][$agent], $agentMatches);
        } else {
          $remapped[$projectedId][$agent] = $agentMatches;
        }
      }
    }
    return $remapped;
  }
 
  private function isRegionIncluded($small, $big)
  {
    return ($big[0] >= 0) && ($small[0] >= $big[0]) && ($small[1] <= $big[1]);
  }
 
  private function areNomosMatchesInsideAMonkMatch($licenseMatches)
  {
    if (!array_key_exists("nomos", $licenseMatches)) {
      return false;
    }
    if (!array_key_exists("monk", $licenseMatches)) {
      return false;
    }
 
    foreach ($licenseMatches["nomos"] as $licenseMatch) {
      $matchId = $licenseMatch->getLicenseFileId();
      $nomosRegion = $this->highlightDao->getHighlightRegion($matchId);
 
      $found = false;
      foreach ($licenseMatches["monk"] as $monkLicenseMatch) {
        $monkRegion = $this->highlightDao->getHighlightRegion($monkLicenseMatch->getLicenseFileId());
        if ($this->isRegionIncluded($nomosRegion, $monkRegion)) {
          $found = true;
          break;
        }
      }
      if (!$found) {
        return false;
      }
    }
 
    return true;
  }
 
  protected function areNomosMonkNinkaAgreed($licenseMatches)
  {
    $scanners = array('nomos','monk','ninka');
    $vote = array();
    foreach ($scanners as $scanner) {
      if (!array_key_exists($scanner, $licenseMatches)) {
        return false;
      }
      foreach ($licenseMatches[$scanner] as $licenseMatch) {
        $licId = $licenseMatch->getLicenseId();
        $vote[$licId][$scanner] = true;
      }
    }
 
    foreach ($vote as $licId=>$voters) {
      if (count($voters) != 3) {
        return false;
      }
    }
    return true;
  }
 
  protected function getLicenseIdsOfMatchesForScanner($scanner, $licenseMatches)
  {
    if (array_key_exists($scanner, $licenseMatches) === true) {
      return array_map(
        function ($match) {
          return $match->getLicenseId();
        }, $licenseMatches[$scanner]);
    }
    return [];
  }
 
  protected function areOtherScannerFindingsAndOJOAgreed($licenseMatches)
  {
    $findingsByOjo = $this->getLicenseIdsOfMatchesForScanner('ojo', $licenseMatches);
    if (count($findingsByOjo) == 0) {
      // nothing to do
      return false;
    }
 
    $findingsByOtherScanner = $this->getLicenseIdsOfMatchesForScanner('nomos', $licenseMatches);
    if (count($findingsByOtherScanner) == 0) {
      // nothing found by other scanner, so no contradiction
      return true;
    }
    foreach ($findingsByOtherScanner as $findingsByScanner) {
      if (in_array($findingsByScanner, $findingsByOjo) === false) {
        // contradiction found
        return false;
      }
    }
    return true;
  }
 
  protected function areOtherScannerFindingsAndRESOAgreed($licenseMatches)
  {
    $findingsByReso = $this->getLicenseIdsOfMatchesForScanner('reso', $licenseMatches);
    if (count($findingsByReso) == 0) {
      // nothing to do
      return false;
    }
 
    $findingsByOtherScanner = $this->getLicenseIdsOfMatchesForScanner('nomos', $licenseMatches);
    if (count($findingsByOtherScanner) == 0) {
      // nothing found by other scanner, so no contradiction
      return true;
    }
    foreach ($findingsByOtherScanner as $findingsByScanner) {
      if (in_array($findingsByScanner, $findingsByReso) === false) {
        // contradiction found
        return false;
      }
    }
    return true;
  }
 
  private function getCopyrightsToDisableFalsePositivesClutter($uploadId,
                                                               $clutter_flag): void
  {
    if (empty($uploadId)) {
      return;
    }
    $agentName = 'copyright';
    $uploadTreeTableName = $this->uploadDao->getUploadtreeTableName($uploadId);
    $scanJobProxy = new ScanJobProxy($GLOBALS['container']->get('dao.agent'), $uploadId);
    $scanJobProxy->createAgentStatus(array($agentName));
    $selectedScanners = $scanJobProxy->getLatestSuccessfulAgentIds();
    if (!array_key_exists($agentName, $selectedScanners)) {
      return;
    }
    $latestXpAgentId = $selectedScanners[$agentName];
    $extrawhere = ' agent_fk='.$latestXpAgentId;
    $allCopyrights = $this->copyrightDao->getScannerEntries('copyright',
      $uploadTreeTableName, $uploadId, null, $extrawhere);
 
    if (empty($allCopyrights)) {
      echo "No copyrights found for upload $uploadId. Skipping false positive deactivation.\n";
      return;
    }
 
    $copyrightJSON = json_encode($allCopyrights);
    $tmpFile = tmpfile();
    $tmpFilePath = stream_get_meta_data($tmpFile)['uri'];
    fwrite($tmpFile, $copyrightJSON);
    $deactivatedCopyrightData = $this->callCopyrightDeactivationClutterRemovalScript($tmpFilePath, $clutter_flag);
    if (empty($deactivatedCopyrightData)) {
      fclose($tmpFile);
      return;
    }
    $deactivatedCopyrights = json_decode($deactivatedCopyrightData, true);
    foreach ($deactivatedCopyrights as $deactivatedCopyright) {
      $item = $deactivatedCopyright['uploadtree_pk'];
      $itemTreeBounds = $this->uploadDao->getItemTreeBounds($item, $uploadTreeTableName);
      $hash = $deactivatedCopyright['hash'];
      $content = $deactivatedCopyright['content'];
      $cpTable = 'copyright';
      if ($deactivatedCopyright['is_copyright'] == "t") {
        $action = '';
        if (array_key_exists('decluttered_content', $deactivatedCopyright) &&
            !empty($deactivatedCopyright['decluttered_content'])) {
          $content = $deactivatedCopyright['decluttered_content'];
        } else {
          // No text update. Nothing to do.
          $this->heartbeat(1);
          continue;
        }
      } else {
        $action = 'delete';
      }
      $this->copyrightDao->updateTable($itemTreeBounds, $hash, $content, $this->userId, $cpTable, $action);
      $this->heartbeat(1);
    }
    fclose($tmpFile);
  }
 
  private function callCopyrightDeactivationClutterRemovalScript($tmpFilePath,
                                                                 $clutter_flag): string
  {
    $script = "copyrightDeactivationClutterRemovalScript.py";
    $args = ["python3", __DIR__ . "/$script", "--file", $tmpFilePath];
    if ($clutter_flag) {
      $args[] = "--clutter";
    }
 
    $sleepTime = 5;
    $maxSleepTime = 25;
 
    $process = new Process($args);
    $process->setTimeout(null); // Remove timeout to run indefinitely.
    $process->setEnv(set_python_path());
    $process->start();
 
    do {
      $this->heartbeat(0);
      sleep($sleepTime);
      if ($sleepTime < $maxSleepTime) {
        $sleepTime += 5;
      }
    } while ($process->isRunning());
 
    echo $process->getErrorOutput();
 
    return $process->getOutput();
  }
 
  private function getLicenseType($licenseType)
  {
    global $SysConf;
    $licenseTypes = array_map('trim', explode(',',
        $SysConf['SYSCONFIG']['LicenseTypes']));
    if (in_array($licenseType, $licenseTypes)) {
      return $licenseType;
    }
    return "";
  }
 
  protected function noLicenseConflict($itemTreeBounds, $licenseMatches)
  {
    $shortnames = [];
    foreach ($licenseMatches as $agentMatch) {
      foreach ($agentMatch as $agentLicenseMatches) {
        foreach ($agentLicenseMatches as $licenseMatch) {
          $shortnames[$licenseMatch->getLicenseId()] = $licenseMatch
              ->getLicenseRef()->getShortName();
        }
      }
    }
    foreach ($shortnames as $shortname) {
      try {
        if (! $this->compatibilityDao->getCompatibilityForFile($itemTreeBounds,
            $shortname)) {
          return false;
        }
      } catch (InvalidAgentStageException $_) {
        return false;
      }
    }
    return true;
  }
 
  protected function allLicenseInType($licenseMatches)
  {
    $licenseTypes = [];
    foreach ($licenseMatches as $agentMatch) {
      foreach ($agentMatch as $agentLicenseMatches) {
        foreach ($agentLicenseMatches as $licenseMatch) {
          if (!array_key_exists($licenseMatch->getLicenseId(), $licenseTypes)) {
            $licenseTypes[$licenseMatch->getLicenseId()] = $this->licenseDao
                ->getLicenseType($licenseMatch->getLicenseId());
          }
        }
      }
    }
    if (! in_array($this->licenseType, $licenseTypes)) {
      return false;
    }
    if (! empty(array_diff($licenseTypes, [$this->licenseType]))) {
      return false;
    }
    return true;
  }
}