schedulerTest.php

Go to the documentation of this file.

<?php
/*
 SPDX-FileCopyrightText: © 2014-2015, 2019 Siemens AG
 
 SPDX-License-Identifier: GPL-2.0-only
*/
namespace Fossology\Reuser\Test;
 
use Fossology\Lib\BusinessRules\ClearingDecisionFilter;
use Fossology\Lib\Dao\AgentDao;
use Fossology\Lib\Dao\ClearingDao;
use Fossology\Lib\Dao\CopyrightDao;
use Fossology\Lib\Dao\HighlightDao;
use Fossology\Lib\Dao\LicenseDao;
use Fossology\Lib\Dao\TreeDao;
use Fossology\Lib\Dao\UploadDao;
use Fossology\Lib\Dao\UploadPermissionDao;
use Fossology\Lib\Data\Clearing\ClearingEvent;
use Fossology\Lib\Data\Clearing\ClearingEventTypes;
use Fossology\Lib\Data\Clearing\ClearingLicense;
use Fossology\Reuser\ReuserAgentPlugin;
use Fossology\Lib\Data\ClearingDecision;
use Fossology\Lib\Data\DecisionScopes;
use Fossology\Lib\Data\DecisionTypes;
use Fossology\Lib\Db\DbManager;
use Fossology\Lib\Test\TestInstaller;
use Fossology\Lib\Test\TestPgDb;
use Monolog\Logger;
 
include_once(__DIR__.'/../../../lib/php/Test/Agent/AgentTestMockHelper.php');
include_once(__DIR__.'/SchedulerTestRunnerCli.php');
include_once(__DIR__.'/SchedulerTestRunnerMock.php');
 
class SchedulerTest extends \PHPUnit\Framework\TestCase
{
  private $groupId = 3;
  private $userId = 2;
  private $testDb;
  private $dbManager;
  private $testInstaller;
  private $licenseDao;
  private $clearingDao;
  private $copyrightDao;
  private $clearingDecisionFilter;
  private $uploadDao;
  private $uploadPermDao;
  private $highlightDao;
  private $treeDao;
 
  private $runnerCli;
 
  private $runnerMock;
 
  protected function setUp() : void
  {
    $this->testDb = new TestPgDb("reuserSched");
    $this->dbManager = $this->testDb->getDbManager();
 
    $this->licenseDao = new LicenseDao($this->dbManager);
    $logger = new Logger("ReuserSchedulerTest");
    $this->uploadPermDao = \Mockery::mock(UploadPermissionDao::class);
    $this->uploadDao = new UploadDao($this->dbManager, $logger, $this->uploadPermDao);
    $this->highlightDao = new HighlightDao($this->dbManager);
    $this->clearingDecisionFilter = new ClearingDecisionFilter();
    $this->clearingDao = new ClearingDao($this->dbManager, $this->uploadDao);
    $this->copyrightDao = new CopyrightDao($this->dbManager, $this->uploadDao);
    $this->treeDao = \Mockery::mock(TreeDao::class);
 
    $agentDao = new AgentDao($this->dbManager, $logger);
 
    $this->runnerMock = new SchedulerTestRunnerMock($this->dbManager, $agentDao,
                        $this->clearingDao, $this->uploadDao, $this->clearingDecisionFilter,
                        $this->treeDao, $this->copyrightDao);
    $this->runnerCli = new SchedulerTestRunnerCli($this->testDb);
  }
 
  protected function tearDown() : void
  {
    $this->testDb->fullDestruct();
    $this->testDb = null;
    $this->dbManager = null;
    $this->licenseDao = null;
    $this->highlightDao = null;
    $this->clearingDao = null;
    $this->copyrightDao = null;
  }
 
  private function setUpRepo()
  {
    $sysConf = $this->testDb->getFossSysConf();
    $this->testInstaller = new TestInstaller($sysConf);
    $this->testInstaller->init();
    $this->testInstaller->cpRepo();
  }
 
  private function rmRepo()
  {
    $this->testInstaller->rmRepo();
    $this->testInstaller->clear();
  }
 
  private function setUpTables()
  {
    $this->testDb->createPlainTables(array('upload','upload_reuse','uploadtree',
      'uploadtree_a','license_ref','license_ref_bulk','clearing_decision',
      'clearing_decision_event','clearing_event','license_file','highlight',
      'highlight_bulk','agent','pfile','ars_master','users','group_user_member',
      'upload_clearing_license','report_info'),false);
    $this->testDb->createSequences(array('agent_agent_pk_seq','pfile_pfile_pk_seq',
      'upload_upload_pk_seq','nomos_ars_ars_pk_seq','license_file_fl_pk_seq',
      'license_ref_rf_pk_seq','license_ref_bulk_lrb_pk_seq',
      'clearing_decision_clearing_decision_pk_seq',
      'clearing_event_clearing_event_pk_seq','report_info_pk_seq'),false);
    $this->testDb->createViews(array('license_file_ref'),false);
    $this->testDb->createConstraints(array('agent_pkey','pfile_pkey',
      'upload_pkey_idx','FileLicense_pkey','clearing_event_pkey'),false);
    $this->testDb->alterTables(array('agent','pfile','upload','ars_master',
      'license_ref_bulk','license_ref','clearing_event','clearing_decision','license_file','highlight'),false);
    $this->testDb->createInheritedTables();
    $this->testDb->createInheritedArsTables(array('monk'));
 
    $this->testDb->insertData(array('pfile','upload','uploadtree_a','users',
      'group_user_member','agent','license_file','monk_ars','report_info'),
      false);
    $this->testDb->insertData_license_ref(80);
 
    $this->testDb->resetSequenceAsMaxOf('agent_agent_pk_seq', 'agent', 'agent_pk');
 
    $this->testDb->setupSysconfig();
  }
 
  private function getHeartCount($output)
  {
    $matches = array();
    if (preg_match("/.*HEART: ([0-9]*).*/", $output, $matches)) {
      return intval($matches[1]);
    } else {
      return -1;
    }
  }
 
  private function getFilteredClearings($uploadId, $groupId)
  {
    $bounds = $this->uploadDao->getParentItemBounds($uploadId);
    return $this->clearingDao->getFileClearingsFolder($bounds, $groupId);
  }
 
  public function testReuserMockedScanWithoutAnyUploadToCopyAndNoClearing()
  {
    $this->runnerReuserScanWithoutAnyUploadToCopyAndNoClearing($this->runnerMock);
  }
 
  public function testReuserRealScanWithoutAnyUploadToCopyAndNoClearing()
  {
    $this->runnerReuserScanWithoutAnyUploadToCopyAndNoClearing($this->runnerCli);
  }
 
  private function runnerReuserScanWithoutAnyUploadToCopyAndNoClearing(SchedulerTestRunner $runner)
  {
    $this->setUpTables();
    $this->setUpRepo();
 
    list($success, $output,$retCode) = $runner->run($uploadId=1, $this->userId);
 
    $this->assertTrue($success, 'cannot run runner');
    $this->assertEquals($retCode, 0, 'reuser failed: '.$output);
 
    assertThat($this->getHeartCount($output), equalTo(0));
 
    $bounds = $this->uploadDao->getParentItemBounds($uploadId);
    assertThat($this->clearingDao->getFileClearingsFolder($bounds, $groupId=5), is(emptyArray()));
 
    $this->rmRepo();
  }
 
  protected function insertDecisionFromTwoEvents($scope=DecisionScopes::ITEM,$originallyClearedItemId=23)
  {
    $licenseRef1 = $this->licenseDao->getLicenseByShortName("SPL-1.0")->getRef();
    $licenseRef2 = $this->licenseDao->getLicenseByShortName("Glide")->getRef();
 
    $addedLicenses = array($licenseRef1, $licenseRef2);
    assertThat($addedLicenses, not(arrayContaining(null)));
 
    $clearingLicense1 = new ClearingLicense($licenseRef1, false, ClearingEventTypes::USER, "42", "44");
    $clearingLicense2 = new ClearingLicense($licenseRef2, true, ClearingEventTypes::USER, "-42", "-44");
 
    $eventId1 = $this->clearingDao->insertClearingEvent($originallyClearedItemId, $this->userId, $this->groupId,
            $licenseRef1->getId(), $clearingLicense1->isRemoved(),
            $clearingLicense1->getType(), $clearingLicense1->getReportinfo(), $clearingLicense1->getComment());
    $eventId2 = $this->clearingDao->insertClearingEvent($originallyClearedItemId, 5, $this->groupId,
            $licenseRef2->getId(), $clearingLicense2->isRemoved(),
            $clearingLicense2->getType(), $clearingLicense2->getReportinfo(), $clearingLicense2->getComment());
 
    $addedEventIds = array($eventId1, $eventId2);
 
    $this->clearingDao->createDecisionFromEvents($originallyClearedItemId, $this->userId,
      $this->groupId, DecisionTypes::IDENTIFIED, $scope, $addedEventIds);
 
    return array($clearingLicense1, $clearingLicense2, $addedEventIds);
  }
 
  public function testReuserMockedScanWithoutAnyUploadToCopyAndAClearing()
  {
    $this->runnerReuserScanWithoutAnyUploadToCopyAndAClearing($this->runnerMock);
  }
 
  public function testReuserRealScanWithoutAnyUploadToCopyAndAClearing()
  {
    $this->runnerReuserScanWithoutAnyUploadToCopyAndAClearing($this->runnerCli);
  }
 
  private function runnerReuserScanWithoutAnyUploadToCopyAndAClearing($runner)
  {
    $this->setUpTables();
    $this->setUpRepo();
 
    $this->insertDecisionFromTwoEvents();
 
    list($success,$output,$retCode) = $runner->run($uploadId=3);
 
    $this->assertTrue($success, 'cannot run runner');
    $this->assertEquals($retCode, 0, 'reuser failed: '.$output);
 
    assertThat($this->getHeartCount($output), equalTo(0));
 
    $decisions = $this->getFilteredClearings($uploadId, $this->groupId);
    assertThat($decisions, is(emptyArray()));
 
    $this->rmRepo();
  }
 
  public function testReuserMockedScanWithALocalClearing()
  {
    $this->runnerReuserScanWithALocalClearing($this->runnerMock,1);
  }
 
  public function testReuserRealScanWithALocalClearing()
  {
    $this->runnerReuserScanWithALocalClearing($this->runnerCli,1);
  }
 
  private function runnerReuserScanWithALocalClearing($runner, $heartBeat=0)
  {
    $this->setUpTables();
    $this->setUpRepo();
 
    $this->uploadDao->addReusedUpload($uploadId=3,$reusedUpload=2,$this->groupId,$this->groupId);
 
    list($clearingLicense1, $clearingLicense2, $addedEventIds) = $this->insertDecisionFromTwoEvents();
 
    /* upload 3 in the test db is the same as upload 2
     * items 13-24 in upload 2 correspond to 33-44 */
    $reusingUploadItemShift = 20;
 
    list($success,$output,$retCode) = $runner->run($uploadId, $this->userId, $this->groupId);
 
    $this->assertTrue($success, 'cannot run runner');
    $this->assertEquals($retCode, 0, 'reuser failed: '.$output);
    assertThat($this->getHeartCount($output), equalTo($heartBeat));
 
    $newUploadClearings = $this->getFilteredClearings($uploadId, $this->groupId);
    $potentiallyReusableClearings = $this->getFilteredClearings($reusedUpload, $this->groupId);
 
    assertThat($newUploadClearings, is(arrayWithSize(1)));
 
    assertThat($potentiallyReusableClearings, is(arrayWithSize(1)));
    $potentiallyReusableClearing = $potentiallyReusableClearings[0];
    $newClearing = $newUploadClearings[0];
 
    assertThat($newClearing, not(equalTo($potentiallyReusableClearing)));
    assertThat($newClearing->getClearingId(), not(equalTo($potentiallyReusableClearing->getClearingId())));
 
    assertThat($newClearing->getClearingLicenses(), arrayContainingInAnyOrder($clearingLicense1, $clearingLicense2));
 
    assertThat($newClearing->getType(), equalTo($potentiallyReusableClearing->getType()));
    assertThat($newClearing->getScope(), equalTo($potentiallyReusableClearing->getScope()));
 
    assertThat($newClearing->getUploadTreeId(),
      equalTo($potentiallyReusableClearing->getUploadTreeId() + $reusingUploadItemShift));
 
    $this->rmRepo();
  }
 
  public function testReuserMockedScanWithARepoClearing()
  {
    $this->runnerReuserScanWithARepoClearing($this->runnerMock);
  }
 
  public function testReuserRealScanWithARepoClearing()
  {
    $this->runnerReuserScanWithARepoClearing($this->runnerCli);
  }
 
  private function runnerReuserScanWithARepoClearing($runner)
  {
    $this->setUpTables();
    $this->setUpRepo();
 
    $this->uploadDao->addReusedUpload($uploadId=3,$reusedUpload=2,$this->groupId,$this->groupId);
 
    list($clearingLicense1, $clearingLicense2, $addedEventIds) = $this->insertDecisionFromTwoEvents(
      DecisionScopes::REPO,$originallyClearedItemId=23);
    $clearingLicenses = array($clearingLicense1, $clearingLicense2);
 
    /* upload 3 in the test db is the same as upload 2
     * items 13-24 in upload 2 correspond to 33-44 */
    $reusingUploadItemShift = 20;
 
    list($success,$output,$retCode) = $runner->run($uploadId, $this->userId, $this->groupId);
 
    $this->assertTrue($success, 'cannot run runner');
    $this->assertEquals($retCode, 0, 'reuser failed: '.$output);
 
    assertThat($this->getHeartCount($output), equalTo(0));
 
    $newUploadClearings = $this->getFilteredClearings($uploadId, $this->groupId);
    $potentiallyReusableClearings = $this->getFilteredClearings($reusedUpload, $this->groupId);
 
    assertThat($newUploadClearings, is(arrayWithSize(1)));
 
    assertThat($potentiallyReusableClearings, is(arrayWithSize(1)));
    $potentiallyReusableClearing = $potentiallyReusableClearings[0];
    $newClearing = $newUploadClearings[0];
 
    /* they are actually the same ClearingDecision
     * only sameFolder and sameUpload are different */
    assertThat($newClearing, not(equalTo($potentiallyReusableClearing)));
 
    /* reuser should have not created a new clearing decision */
    assertThat($newClearing->getClearingId(), equalTo($potentiallyReusableClearing->getClearingId()));
 
    assertThat($newClearing->getClearingLicenses(), arrayContainingInAnyOrder($clearingLicenses));
 
    assertThat($newClearing->getType(), equalTo($potentiallyReusableClearing->getType()));
    assertThat($newClearing->getScope(), equalTo($potentiallyReusableClearing->getScope()));
 
    assertThat($newClearing->getUploadTreeId(),
            equalTo($potentiallyReusableClearing->getUploadTreeId() + $reusingUploadItemShift));
 
    /* reuser should have not created a correct local event history */
    $bounds = $this->uploadDao->getItemTreeBounds($originallyClearedItemId + $reusingUploadItemShift);
    $newEvents = $this->clearingDao->getRelevantClearingEvents($bounds, $this->groupId);
 
    assertThat($newEvents, is(arrayWithSize(count($clearingLicenses))));
 
    foreach ($newEvents as $newEvent) {
      assertThat($newEvent->getEventId(), anyOf($addedEventIds));
      assertThat($newEvent->getClearingLicense(), anyOf($clearingLicenses));
    }
 
    $this->rmRepo();
  }
 
  public function testReuserRealScanWithARepoClearingEnhanced()
  {
    $this->runnerReuserScanWithARepoClearingEnhanced($this->runnerMock);
  }
 
  private function runnerReuserScanWithARepoClearingEnhanced($runner)
  {
    $this->setUpTables();
    $this->setUpRepo();
 
    $originallyClearedItemId = 23;
    /* upload 3 in the test db is the same as upload 2 -> items 13-24 in upload 2 correspond to 33-44 */
    $reusingUploadItemShift = 20;
 
    $this->uploadDao->addReusedUpload($uploadId=3,$reusedUpload=2,$this->groupId,$this->groupId,$reuseMode=2);
 
    $repoPath = $this->testDb->getFossSysConf().'/repo/files/';
    $this->treeDao->shouldReceive('getRepoPathOfPfile')->with(4)->andReturn($repoPath
      .'04621571bcbabce75c4dd1c6445b87dec0995734.59cacdfce5051cd8a1d8a1f2dcce40a5.12320');
    $this->treeDao->shouldReceive('getRepoPathOfPfile')->with(351)->andReturn($repoPath
      .'c518ce1658140b65fa0132ad1130cb91512416bf.8e913e594d24ff3aeabe350107d97815.35829');
 
    list($clearingLicense1, $clearingLicense2, $addedEventIds) = $this->insertDecisionFromTwoEvents(
      DecisionScopes::REPO,$originallyClearedItemId);
    $clearingLicenses = array($clearingLicense1, $clearingLicense2);
 
    list($success,$output,$retCode) = $runner->run($uploadId, $this->userId, $this->groupId);
 
    $this->assertTrue($success, 'cannot run runner');
    $this->assertEquals($retCode, 0, 'reuser failed: '.$output);
 
    $newUploadClearings = $this->getFilteredClearings($uploadId, $this->groupId);
    $potentiallyReusableClearings = $this->getFilteredClearings($reusedUpload, $this->groupId);
 
    assertThat($newUploadClearings, is(arrayWithSize(1)));
 
    assertThat($potentiallyReusableClearings, is(arrayWithSize(1)));
    $potentiallyReusableClearing = $potentiallyReusableClearings[0];
    $newClearing = $newUploadClearings[0];
 
    /* they are actually the same ClearingDecision
     * only sameFolder and sameUpload are different */
    assertThat($newClearing, not(equalTo($potentiallyReusableClearing)));
 
    assertThat($newClearing->getClearingLicenses(), arrayContainingInAnyOrder($clearingLicenses));
 
    assertThat($newClearing->getType(), equalTo($potentiallyReusableClearing->getType()));
    assertThat($newClearing->getScope(), equalTo($potentiallyReusableClearing->getScope()));
 
    assertThat($newClearing->getUploadTreeId(),
            equalTo($potentiallyReusableClearing->getUploadTreeId() + $reusingUploadItemShift));
 
    /* reuser should have not created a correct local event history */
    $bounds = $this->uploadDao->getItemTreeBounds($originallyClearedItemId + $reusingUploadItemShift);
    $newEvents = $this->clearingDao->getRelevantClearingEvents($bounds, $this->groupId);
 
    assertThat($newEvents, is(arrayWithSize(count($clearingLicenses))));
 
    foreach ($newEvents as $newEvent) {
      assertThat($newEvent->getEventId(), anyOf($addedEventIds));
      assertThat($newEvent->getClearingLicense(), anyOf($clearingLicenses));
    }
    /*reuse main license*/
    $this->clearingDao->makeMainLicense($uploadId=2, $this->groupId, $mainLicenseId=402);
    $mainLicenseIdForReuse = $this->clearingDao->getMainLicenseIds($reusedUploadId=2, $this->groupId);
    $mainLicenseIdForReuseSingle = array_values($mainLicenseIdForReuse);
    $this->clearingDao->makeMainLicense($uploadId=3, $this->groupId, $mainLicenseIdForReuseSingle[0]);
    $mainLicense=$this->clearingDao->getMainLicenseIds($uploadId=3, $this->groupId);
    $mainLicenseSingle = array_values($mainLicense);
    $this->assertEquals($mainLicenseIdForReuseSingle, $mainLicenseSingle);
    $this->rmRepo();
  }
 
  public function testReuserMultipleReuseSelectionsValidation()
  {
    // Test array format (multiple selections)
    $reuseSelections = ['2,1', '4,1'];
    // Simulate the validation logic from scheduleAgent
    $createdLinks = 0;
    foreach ($reuseSelections as $reuseSelection) {
      if (empty($reuseSelection) || !is_string($reuseSelection)) {
        $this->fail("Invalid reuse selection found - empty or non-string value");
      }
 
      $reuseUploadPair = explode(',', $reuseSelection, 2);
      if (count($reuseUploadPair) !== 2) {
        $this->fail("Invalid reuse selection format: '$reuseSelection' (expected format: 'uploadId,groupId')");
      }
 
      [$reuseUploadId, $reuseGroupId] = $reuseUploadPair;
      $this->assertIsNumeric($reuseUploadId, "Upload ID should be numeric");
      $this->assertIsNumeric($reuseGroupId, "Group ID should be numeric");
      $createdLinks++;
    }
 
    $this->assertEquals(2, $createdLinks, 'Should process 2 reuse selections');
  }
 
  public function testReuserSingleReuseSelectionValidation()
  {
    // Test scalar format (single selection)
    $reuseSelections = '2,1';
    // Simulate the validation logic from scheduleAgent
    if (!is_array($reuseSelections)) {
      $reuseSelections = [$reuseSelections];
    }
 
    $createdLinks = 0;
    foreach ($reuseSelections as $reuseSelection) {
      if (empty($reuseSelection) || !is_string($reuseSelection)) {
        $this->fail("Invalid reuse selection found - empty or non-string value");
      }
 
      $reuseUploadPair = explode(',', $reuseSelection, 2);
      if (count($reuseUploadPair) !== 2) {
        $this->fail("Invalid reuse selection format: '$reuseSelection' (expected format: 'uploadId,groupId')");
      }
 
      [$reuseUploadId, $reuseGroupId] = $reuseUploadPair;
      $this->assertIsNumeric($reuseUploadId, "Upload ID should be numeric");
      $this->assertIsNumeric($reuseGroupId, "Group ID should be numeric");
      $createdLinks++;
    }
 
    $this->assertEquals(1, $createdLinks, 'Should process 1 reuse selection');
  }
 
  public function testReuserInvalidReuseSelectionFormat()
  {
    $invalidSelection = "invalid_format";
    // Simulate the validation logic from scheduleAgent
    $reuseUploadPair = explode(',', $invalidSelection, 2);
    $this->expectException(\InvalidArgumentException::class);
    $this->expectExceptionMessage("Reuser: Invalid reuse selection format: '$invalidSelection' (expected format: 'uploadId,groupId')");
 
    if (count($reuseUploadPair) !== 2) {
      throw new \InvalidArgumentException("Reuser: Invalid reuse selection format: '$invalidSelection' (expected format: 'uploadId,groupId')");
    }
  }
 
  public function testReuserEmptyReuseSelection()
  {
    $emptySelection = "";
    $this->expectException(\InvalidArgumentException::class);
    $this->expectExceptionMessage("Reuser: Invalid reuse selection found - empty or non-string value");
 
    if (empty($emptySelection) || !is_string($emptySelection)) {
      throw new \InvalidArgumentException("Reuser: Invalid reuse selection found - empty or non-string value");
    }
  }
 
  public function testReuserNonStringReuseSelection()
  {
    $nonStringSelection = 123;
    $this->expectException(\InvalidArgumentException::class);
    $this->expectExceptionMessage("Reuser: Invalid reuse selection found - empty or non-string value");
 
    if (empty($nonStringSelection) || !is_string($nonStringSelection)) {
      throw new \InvalidArgumentException("Reuser: Invalid reuse selection found - empty or non-string value");
    }
  }
 
  public function testReuserMalformedReuseSelection()
  {
    // Test various malformed formats
    $malformedCases = [
      '123,',    // Missing group ID
      ',456',    // Missing upload ID
      '123,456,789', // Too many parts
      '123'      // Missing comma separator
    ];
 
    foreach ($malformedCases as $malformedValue) {
      $reuseUploadPair = explode(',', $malformedValue, 2);
      if (count($reuseUploadPair) !== 2) {
        $this->expectException(\InvalidArgumentException::class);
        $this->expectExceptionMessage("Reuser: Invalid reuse selection format: '$malformedValue' (expected format: 'uploadId,groupId')");
        throw new \InvalidArgumentException("Reuser: Invalid reuse selection format: '$malformedValue' (expected format: 'uploadId,groupId')");
      }
    }
  }
}