spdx.php

Go to the documentation of this file.

<?php
/*
 SPDX-FileCopyrightText: © 2015-2016,2023 Siemens AG
 SPDX-FileCopyrightText: © 2017 TNG Technology Consulting GmbH
 
 SPDX-License-Identifier: GPL-2.0-only
*/
namespace Fossology\Spdx;
 
use Fossology\Lib\Agent\Agent;
use Fossology\Lib\Auth\Auth;
use Fossology\Lib\BusinessRules\LicenseMap;
use Fossology\Lib\Dao\ClearingDao;
use Fossology\Lib\Dao\LicenseDao;
use Fossology\Lib\Dao\TreeDao;
use Fossology\Lib\Dao\UploadDao;
use Fossology\Lib\Data\AgentRef;
use Fossology\Lib\Data\License;
use Fossology\Lib\Data\LicenseRef;
use Fossology\Lib\Data\Package\ComponentType;
use Fossology\Lib\Data\Report\FileNode;
use Fossology\Lib\Data\Report\SpdxLicenseInfo;
use Fossology\Lib\Data\Upload\Upload;
use Fossology\Lib\Db\DbManager;
use Fossology\Lib\Report\LicenseClearedGetter;
use Fossology\Lib\Report\LicenseMainGetter;
use Fossology\Lib\Report\ObligationsGetter;
use Fossology\Lib\Report\ReportUtils;
use Fossology\Lib\Util\StringOperation;
use Twig\Environment;
 
include_once(__DIR__ . "/spdxutils.php");
 
include_once(__DIR__ . "/version.php");
include_once(__DIR__ . "/services.php");
 
class SpdxAgent extends Agent
{
 
  const OUTPUT_FORMAT_KEY = "outputFormat";               
  const DEFAULT_OUTPUT_FORMAT = "spdx2";                  
  const AVAILABLE_OUTPUT_FORMATS = "spdx2,spdx2tv,dep5,spdx2csv,spdx3jsonld,spdx3json,spdx3rdf,spdx3tv";  
  const UPLOAD_ADDS = "uploadsAdd";                       
  const DATA_LICENSE = "CC0-1.0";                         
 
  private $uploadDao;
  private $clearingDao;
  private $licenseDao;
  protected $dbManager;
  protected $renderer;
  private $licenseMap;
  private $licenseClearedGetter;
  private $licenseMainGetter;
  private $obligationsGetter;
  private $reportutils;
  protected $agentNames = AgentRef::AGENT_LIST;
  protected $filebasename = null;
  protected $uri;
  protected $filename;
  private $licensesInDocument = [];
  protected $outputFormat = self::DEFAULT_OUTPUT_FORMAT;
  protected $concludedLicenseFileIds=[];
  protected $declaredLicenseFileIds=[];
 
  function __construct()
  {
    // deduce the agent name from the command line arguments
    $args = getopt("", array(self::OUTPUT_FORMAT_KEY.'::'));
    $agentName = "";
    if (array_key_exists(self::OUTPUT_FORMAT_KEY, $args)) {
      $agentName = trim($args[self::OUTPUT_FORMAT_KEY]);
    }
    if (empty($agentName)) {
        $agentName = self::DEFAULT_OUTPUT_FORMAT;
    }
 
    parent::__construct($agentName, AGENT_VERSION, AGENT_REV);
 
    $this->uploadDao = $this->container->get('dao.upload');
    $this->clearingDao = $this->container->get('dao.clearing');
    $this->licenseDao = $this->container->get('dao.license');
    $this->dbManager = $this->container->get('db.manager');
    $this->renderer = $this->container->get('twig.environment');
    $this->renderer->setCache(false);
 
    $this->agentSpecifLongOptions[] = self::UPLOAD_ADDS.':';
    $this->agentSpecifLongOptions[] = self::OUTPUT_FORMAT_KEY.':';
 
    $this->licenseClearedGetter = new LicenseClearedGetter();
    $this->licenseMainGetter = new LicenseMainGetter();
    $this->obligationsGetter = new ObligationsGetter();
    $this->reportutils = new ReportUtils();
  }
 
  protected function preWorkOnArgs($args)
  {
    if ((!array_key_exists(self::OUTPUT_FORMAT_KEY, $args)
         || $args[self::OUTPUT_FORMAT_KEY] === "")
        && array_key_exists(self::UPLOAD_ADDS,$args)) {
      $args = SpdxUtils::preWorkOnArgsFlp($args,self::UPLOAD_ADDS,self::OUTPUT_FORMAT_KEY);
    } else {
      if (!array_key_exists(self::UPLOAD_ADDS,$args) || $args[self::UPLOAD_ADDS] === "") {
        $args = SpdxUtils::preWorkOnArgsFlp($args,self::UPLOAD_ADDS,self::OUTPUT_FORMAT_KEY);
      }
    }
    return $args;
  }
 
  function processUploadId($uploadId)
  {
    $args = $this->preWorkOnArgs($this->args);
 
    if (array_key_exists(self::OUTPUT_FORMAT_KEY,$args)) {
      $possibleOutputFormat = trim($args[self::OUTPUT_FORMAT_KEY]);
      if (in_array($possibleOutputFormat, explode(',',self::AVAILABLE_OUTPUT_FORMATS))) {
        $this->outputFormat = $possibleOutputFormat;
      }
    }
    $this->licenseMap = new LicenseMap($this->dbManager, $this->groupId, LicenseMap::REPORT, true);
    $this->computeUri($uploadId);
    $docLicense = $this->licenseDao->getLicenseByShortName(self::DATA_LICENSE);
    $docLicenseId = $docLicense->getId() . "-" . md5($docLicense->getText());
    $this->licensesInDocument[$docLicenseId] = (new SpdxLicenseInfo())
      ->setLicenseObj($docLicense)
      ->setListedLicense(true)
      ->setCustomText(false)
      ->setTextPrinted(true);
    $packageNodes = $this->renderPackage($uploadId);
    $additionalUploadIds = array_key_exists(self::UPLOAD_ADDS,$args) ? explode(',',$args[self::UPLOAD_ADDS]) : array();
    $packageIds = array($uploadId);
    foreach ($additionalUploadIds as $additionalId) {
      $packageNodes .= $this->renderPackage($additionalId);
      $packageIds[] = $additionalId;
    }
    $this->writeReport($packageNodes, $packageIds, $uploadId);
    return true;
  }
 
  protected function getTemplateFile($partname)
  {
    $prefix = $this->outputFormat . "-";
    $postfix = ".twig";
    switch ($this->outputFormat) {
      case "spdx2":
        $postfix = ".xml" . $postfix;
        break;
      case "spdx2csv":
      case "spdx2tv":
        break;
      case "dep5":
        $prefix .= "copyright-";
        break;
      case "spdx3jsonld":
        break;
      case "spdx3rdf":
        $postfix = ".xml" . $postfix;
        break;
      case "spdx3json":
        break;
      case "spdx3tv":
        break;
    }
    return $prefix . $partname . $postfix;
  }
 
  protected function getFileBasename($packageName)
  {
    if ($this->filebasename == null) {
      $fileName = strtoupper($this->outputFormat)."_".$packageName;
      switch ($this->outputFormat) {
        case "spdx2":
          $fileName .= ".spdx.rdf";
          break;
        case "spdx2tv":
          $fileName .= ".spdx";
          break;
        case "spdx2csv":
          $fileName .= ".csv";
          break;
        case "dep5":
          $fileName .= ".txt";
          break;
        case "spdx3jsonld":
          $fileName = $fileName .".jsonld";
          break;
        case "spdx3json":
          $fileName = $fileName .".json";
          break;
        case "spdx3rdf":
          $fileName = $fileName .".spdx.rdf";
          break;
        case "spdx3tv":
          $fileName = $fileName .".spdx";
          break;
      }
      $this->filebasename = $fileName;
    }
    return $this->filebasename;
  }
 
  protected function getFileName($packageName)
  {
    global $SysConf;
    $fileBase = $SysConf['FOSSOLOGY']['path']."/report/";
    return $fileBase. $this->getFileBasename($packageName);
  }
 
  protected function getUri($packageName)
  {
    global $SysConf;
    $url=$SysConf['SYSCONFIG']['FOSSologyURL'];
    if (substr( $url, 0, 4 ) !== "http") {
      $url="http://".$url;
    }
 
    return rtrim($url, '/') . '/' . $this->getFileBasename($packageName);
  }
 
  protected function renderPackage($uploadId)
  {
    $uploadTreeTableName = $this->uploadDao->getUploadtreeTableName($uploadId);
    $stateOsselot = $this->getSPDXReportConf($uploadId, 2);
    $itemTreeBounds = $this->uploadDao->getParentItemBounds($uploadId,$uploadTreeTableName);
    $this->heartbeat(0);
 
    $filesWithLicenses = $this->reportutils
      ->getFilesWithLicensesFromClearings($itemTreeBounds, $this->groupId,
        $this, $this->licensesInDocument);
    $this->heartbeat(0);
 
    $this->reportutils->addClearingStatus($filesWithLicenses,$itemTreeBounds, $this->groupId);
    $this->heartbeat(0);
 
    $scannerIDs = $this->reportutils->addScannerResults($filesWithLicenses, $itemTreeBounds, $this->groupId, $this->licensesInDocument);
    $licenseComment = "";
    if (!empty($scannerIDs)) {
      $licenseComment = $this->getLicenseComment($scannerIDs);
    }
    $this->heartbeat(0);
 
    $this->reportutils->addCopyrightResults($filesWithLicenses, $uploadId);
    $this->heartbeat(0);
 
    $upload = $this->uploadDao->getUpload($uploadId);
    $fileNodes = $this->generateFileNodes($filesWithLicenses, $upload->getTreeTableName(), $uploadId);
 
    $mainLicenseIds = $this->clearingDao->getMainLicenseIds($uploadId, $this->groupId);
    $mainLicenses = array();
    foreach ($mainLicenseIds as $licId) {
      $reportedLicenseId = $this->licenseMap->getProjectedId($licId);
      $mainLicense = $this->licenseDao->getLicenseById($reportedLicenseId, $this->groupId);
      if ($mainLicense === null) {
        error_log(
            "spdx: Warning: main license ID {$reportedLicenseId} not found; skipping."
        );
        continue; // Skip this license and continue with the next one
      }
      $reportLicId = $mainLicense->getId() . "-" . md5($mainLicense->getText());
      $mainLicenses[] = $reportLicId;
      if (!array_key_exists($reportLicId, $this->licensesInDocument)) {
        $listedLicense = stripos($mainLicense->getSpdxId(),
            LicenseRef::SPDXREF_PREFIX) !== 0;
        $this->licensesInDocument[$reportLicId] = (new SpdxLicenseInfo())
          ->setLicenseObj($mainLicense)
          ->setCustomText(false)
          ->setListedLicense($listedLicense);
      }
    }
    $mainLicenseString = [];
    if ($this->outputFormat == "spdx2tv" ||
        $this->outputFormat == "spdx2csv" ||
        $this->outputFormat == "spdx3tv") {
      foreach ($mainLicenses as $mainLicense) {
        $shortName = $this->licensesInDocument[$mainLicense]
          ->getLicenseObj()->getShortName();
        if (StringOperation::stringStartsWith($shortName,
          LicenseRef::SPDXREF_PREFIX)) {
          $mainLicenseString[] = $shortName;
        } else {
          $mainLicenseString[] = $this->licensesInDocument[$mainLicense]
            ->getLicenseObj()->getSpdxId();
        }
      }
      if ($this->outputFormat == "spdx2tv") {
        if ($stateOsselot) {
            $mainLicenseString = SpdxUtils::removeEmptyLicenses($mainLicenseString);
          foreach ($mainLicenseString as $i => $licId) {
            if (StringOperation::stringStartsWith($licId, LicenseRef::SPDXREF_PREFIX)
                  || StringOperation::stringStartsWith($licId, "Dual-license")) {
                continue;
            }
              $mainLicenseString[$i] = "LicenseRef-$licId";
          }
        }
      }
      $mainLicenseString = SpdxUtils::implodeLicenses(
        SpdxUtils::removeEmptyLicenses($mainLicenseString));
    }
 
    $hashes = $this->uploadDao->getUploadHashes($uploadId);
 
    $reportInfo = $this->uploadDao->getReportInfo($uploadId);
    $componentId = $reportInfo['ri_component_id'];
    $componentType = $reportInfo['ri_component_type'];
    $componentVersion = $reportInfo['ri_version'];
    $generalAssessment = $reportInfo['ri_general_assesment'];
    $releaseDate = $reportInfo['ri_release_date'];
    if ($componentId == "NA") {
      $componentId = "";
    }
    if ($componentVersion == "NA") {
      $componentVersion = "";
    }
    if ($generalAssessment == "NA") {
      $generalAssessment = "";
    }
    if ($releaseDate == "NA") {
      $releaseDate = "";
    } else {
      $timeStamp = strtotime($releaseDate);
      if ($timeStamp != -1) {
        $releaseDate = date("Y-m-d\\T00:00:00\\Z", $timeStamp);
      } else {
        $releaseDate = "";
      }
    }
    if ($componentType == ComponentType::MAVEN) {
      $componentType = "maven-central";
    } elseif ($componentType == ComponentType::PACKAGEURL) {
      $componentType = "purl";
    } else {
      if (!empty($componentType)) {
        $componentType = ComponentType::TYPE_MAP[$componentType];
      } else {
        $componentType = ComponentType::TYPE_MAP[ComponentType::PURL];
      }
    }
    $obligations = $this->getObligations($uploadId, $this->groupId);
 
    return $this->renderString($this->getTemplateFile('package'), [
      'packageId' => $uploadId,
      'uri' => $this->uri,
      'packageName' => $upload->getFilename(),
      'packageVersion' => $componentVersion,
      'releaseDate' => $releaseDate,
      'generalAssessment' => $generalAssessment,
      'uploadName' => $upload->getFilename(),
      'componentType' => $componentType,
      'componentId' => htmlspecialchars($componentId),
      'sha1' => $hashes['sha1'],
      'md5' => $hashes['md5'],
      'sha256' => $hashes['sha256'],
      'verificationCode' => $this->getVerificationCode($upload),
      'mainLicenses' => $mainLicenses,
      'mainLicenseString' => $mainLicenseString,
      'licenseComments' => $licenseComment,
      'fileNodes' => $fileNodes,
      'obligations' => $obligations,
      'licenseList' => $this->licensesInDocument,
      'EnableOsselotExport' => $stateOsselot
    ]);
  }
 
  protected function getLicenseComment($scannerIds)
  {
    $agentDao = $this->agentDao;
    $func = function($scannerId) use ($agentDao)
    {
      return $agentDao->getAgentName($scannerId)." (".$agentDao->getAgentRev($scannerId).")";
    };
    $scannerNames = array_map($func, $scannerIds);
    return "licenseInfoInFile determined by Scanners:\n - ".implode("\n - ",$scannerNames);
  }
 
  protected function toLicensesWithFilesAdder(&$filesWithLicenses, $licenses, $copyrights, $file, $fullPath)
  {
    if (!array_key_exists($licenses, $filesWithLicenses)) {
      $filesWithLicenses[$licenses]['files']=array();
      $filesWithLicenses[$licenses]['copyrights']=array();
    }
    if (empty($copyrights)) {
      $copyrights = array();
    }
    $filesWithLicenses[$licenses]['files'][$file] = $fullPath;
    foreach ($copyrights as $copyright) {
      if (!in_array($copyright, $filesWithLicenses[$licenses]['copyrights'])) {
        $filesWithLicenses[$licenses]['copyrights'][] = $copyright;
      }
    }
  }
 
  protected function toLicensesWithFiles(&$filesWithLicenses, $treeTableName)
  {
    $licensesWithFiles = array();
    $treeDao = $this->container->get('dao.tree');
    $filesProceeded = 0;
    foreach ($filesWithLicenses as $fileId => $fileNode) {
      $filesProceeded += 1;
      if (($filesProceeded&2047)==0) {
        $this->heartbeat(0);
      }
      $fullPath = $treeDao->getFullPath($fileId, $treeTableName, 0);
      if (! empty($fileNode->getConcludedLicenses())) {
        $licenses = [];
        foreach ($fileNode->getConcludedLicenses() as $license) {
          $licenses[] = $this->licensesInDocument[$license]
            ->getLicenseObj()->getSpdxId();
        }
        $licenses = SpdxUtils::implodeLicenses(
          SpdxUtils::removeEmptyLicenses(array_unique($licenses)));
        $this->toLicensesWithFilesAdder($licensesWithFiles,
          $licenses, $fileNode->getCopyrights(), $fileId, $fullPath);
      } else {
        if (! empty($fileNode->getScanners())) {
          $implodedLicenses = [];
          foreach ($fileNode->getScanners() as $license) {
            $implodedLicenses[] = $this->licensesInDocument[$license]
              ->getLicenseObj()->getSpdxId();
          }
          $implodedLicenses = SpdxUtils::implodeLicenses(
            SpdxUtils::removeEmptyLicenses(array_unique($implodedLicenses)));
          if ($fileNode->isCleared()) {
            $msgLicense = "None (scanners found: " . $implodedLicenses . ")";
          } else {
            $msgLicense = "NoLicenseConcluded (scanners found: " . $implodedLicenses . ")";
          }
        } else {
          if ($fileNode->isCleared()) {
            $msgLicense = "None";
          } else {
            $msgLicense = "NoLicenseConcluded";
          }
        }
        $this->toLicensesWithFilesAdder($licensesWithFiles, $msgLicense,
          $fileNode->getCopyrights(), $fileId, $fullPath);
      }
    }
    return $licensesWithFiles;
  }
 
  protected function computeUri($uploadId)
  {
    $upload = $this->uploadDao->getUpload($uploadId);
    $packageName = $upload->getFilename();
 
    $this->uri = $this->getUri($packageName);
    $this->filename = $this->getFileName($packageName);
  }
 
  protected function writeReport(&$packageNodes, $packageIds, $uploadId)
  {
    global $SysConf;
    $stateOsselot = $this->getSPDXReportConf($uploadId, 2);
    $fileBase = dirname($this->filename);
 
    if (!is_dir($fileBase)) {
      mkdir($fileBase, 0777, true);
    }
    umask(0133);
 
    $organizationName = $SysConf['SYSCONFIG']["ReportHeaderText"] ?? 'FOSSology';
    $version = $SysConf['BUILD']['VERSION'];
 
    $uploadTreeTableName = $this->uploadDao->getUploadtreeTableName($uploadId);
    $itemTreeBounds = $this->uploadDao->getParentItemBounds($uploadId,$uploadTreeTableName);
    $this->heartbeat(0);
 
    $filesWithLicenses = $this->reportutils
      ->getFilesWithLicensesFromClearings($itemTreeBounds, $this->groupId,
        $this, $this->licensesInDocument);
    $this->heartbeat(0);
    $upload = $this->uploadDao->getUpload($uploadId);
    $this->generateFileNodes($filesWithLicenses, $upload->getTreeTableName(), $uploadId);
    $this->declaredLicenseFileIds = array_unique(array_diff($this->declaredLicenseFileIds, $this->concludedLicenseFileIds));
    $this->concludedLicenseFileIds = array_unique($this->concludedLicenseFileIds);
    $message = $this->renderString($this->getTemplateFile('document'),array(
        'documentName' => $fileBase,
        'uri' => $this->uri,
        'userName' => $this->container->get('dao.user')->getUserName($this->userId) . " (" . $this->container->get('dao.user')->getUserEmail($this->userId) . ")",
        'organisation' => $organizationName,
        'concludedLicenseFileIds'=>$this->concludedLicenseFileIds,
        'declaredLicenseFileIds'=>$this->declaredLicenseFileIds,
        'toolVersion' => 'fossology-' . $version,
        'packageNodes' => $packageNodes,
        'packageIds' => $packageIds,
        'dataLicense' => $this->getSPDXDataLicense(),
        'licenseList' => $this->licensesInDocument,
        'EnableOsselotExport' => $stateOsselot,
      )
    );
 
    // To ensure the file is valid, replace any non-printable characters with a question mark.
    // 'Non-printable' is ASCII < 0x20 (excluding \r, \n and tab) and 0x7F (delete).
    $message = preg_replace('/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/','?',$message);
 
    file_put_contents($this->filename, $message);
    $this->updateReportTable($uploadId, $this->jobId, $this->filename);
  }
 
  protected function updateReportTable($uploadId, $jobId, $fileName)
  {
    $this->reportutils->updateOrInsertReportgenEntry($uploadId, $jobId, $fileName);
  }
 
  protected function renderString($templateName, $vars)
  {
    return $this->renderer->load($templateName)->render($vars);
  }
 
  protected function generateFileNodes($filesWithLicenses, $treeTableName, $uploadId)
  {
    $this->deduplicateLicenseList();
    if (strcmp($this->outputFormat, "dep5") !== 0) {
      return $this->generateFileNodesByFiles($filesWithLicenses, $treeTableName, $uploadId);
    } else {
      return $this->generateFileNodesByLicenses($filesWithLicenses, $treeTableName);
    }
  }
 
  protected function generateFileNodesByFiles($filesWithLicenses, $treeTableName, $uploadId)
  {
    /* @var $treeDao TreeDao */
    $treeDao = $this->container->get('dao.tree');
    $filesProceeded = 0;
    $lastValue = 0;
    $content = '';
    $textToBePrinted = [];
    foreach ($filesWithLicenses as $fileId => $fileData) {
      $filesProceeded += 1;
      if (($filesProceeded & 2047) == 0) {
        $this->heartbeat($filesProceeded - $lastValue);
        $lastValue = $filesProceeded;
      }
      $hashes = $treeDao->getItemHashes($fileId);
      $fileName = $treeDao->getFullPath($fileId, $treeTableName, 0);
      $stateComment = $this->getSPDXReportConf($uploadId, 0);
      $stateWoInfos = $this->getSPDXReportConf($uploadId, 1);
      $stateOsselot = $this->getSPDXReportConf($uploadId, 2);
      foreach ($fileData->getConcludedLicenses() as $license) {
        $this->concludedLicenseFileIds[] = $fileId;
        if (! $this->licensesInDocument[$license]->isTextPrinted()) {
          $textToBePrinted[] = $license;
        }
      }
      foreach ($fileData->getScanners() as $license) {
        $this->declaredLicenseFileIds[] = $fileId;
        if (! $this->licensesInDocument[$license]->isTextPrinted()) {
          $textToBePrinted[] = $license;
        }
      }
      $concludedLicensesString = [];
      if ($this->outputFormat == "spdx2tv" ||
          $this->outputFormat == "spdx2csv" ||
          $this->outputFormat == "spdx3tv") {
        foreach ($fileData->getConcludedLicenses() as $license) {
          $shortName = $this->licensesInDocument[$license]
            ->getLicenseObj()->getShortName();
          if (StringOperation::stringStartsWith($shortName,
            LicenseRef::SPDXREF_PREFIX)) {
            $concludedLicensesString[] = $shortName;
          } else {
            $concludedLicensesString[] = $this->licensesInDocument[$license]
              ->getLicenseObj()->getSpdxId();
          }
        }
        if ($this->outputFormat == "spdx2tv") {
          if ($stateOsselot) {
            foreach ($concludedLicensesString as $j => $licId) {
              if (StringOperation::stringStartsWith($licId, LicenseRef::SPDXREF_PREFIX)
                    || StringOperation::stringStartsWith($licId, "Dual-license")) {
                  continue;
              }
                $concludedLicensesString[$j] = "LicenseRef-$licId";
            }
          }
          $concludedLicensesString = SpdxUtils::implodeLicenses(
              SpdxUtils::removeEmptyLicenses($concludedLicensesString)
          );
        } else {
          $concludedLicensesString = SpdxUtils::implodeLicenses(
            SpdxUtils::removeEmptyLicenses($concludedLicensesString));
        }
      }
      if (!$stateWoInfos ||
          ($stateWoInfos && (!empty($fileData->getConcludedLicenses()) ||
              !empty($fileData->getScanners()) || !empty($fileData->getCopyrights())))) {
        $fileData->setAcknowledgements(
          SpdxUtils::cleanTextArray($fileData->getAcknowledgements()));
        $fileData->setComments(
          SpdxUtils::cleanTextArray($fileData->getComments()));
        $dataTemplate = array(
          'fileId' => $fileId,
          'sha1' => $hashes['sha1'],
          'md5' => $hashes['md5'],
          'sha256' => $hashes['sha256'],
          'uri' => $this->uri,
          'fileName' => $fileName,
          'fileDirName' => dirname($fileName),
          'fileBaseName' => basename($fileName),
          'fileData' => $fileData,
          'licenseList' => $this->licensesInDocument,
          'concludedLicensesString' => $concludedLicensesString,
          'licenseCommentState' => $stateComment,
          'EnableOsselotExport' =>$stateOsselot,
        );
        $content .= $this->renderString($this->getTemplateFile('file'),
          $dataTemplate);
      }
      foreach ($textToBePrinted as $license) {
        $this->licensesInDocument[$license]->setTextPrinted(true);
      }
    }
    $this->heartbeat($filesProceeded - $lastValue);
    return $content;
  }
 
  protected function generateFileNodesByLicenses($filesWithLicenses, $treeTableName)
  {
    $licensesWithFiles = $this->toLicensesWithFiles($filesWithLicenses, $treeTableName);
 
    $content = '';
    $filesProceeded = 0;
    $lastStep = 0;
    $lastValue = 0;
    foreach ($licensesWithFiles as $licenseId=>$entry) {
      $filesProceeded += count($entry['files']);
      if ($filesProceeded&(~2047) > $lastStep) {
        $this->heartbeat($filesProceeded - $lastValue);
        $lastStep = $filesProceeded&(~2047) + 2048;
        $lastValue = $filesProceeded;
      }
 
      $comment = "";
      if (strrpos($licenseId, "NoLicenseConcluded (scanners found: ", -strlen($licenseId)) !== false) {
        $comment = substr($licenseId,20,strlen($licenseId)-21);
        $licenseId = "NoLicenseConcluded";
      } elseif (strrpos($licenseId, "None (scanners found: ", -strlen($licenseId)) !== false) {
        $comment = substr($licenseId,6,strlen($licenseId)-7);
        $licenseId = "None";
      }
 
      $content .= $this->renderString($this->getTemplateFile('file'),array(
          'fileNames'=>$entry['files'],
          'license'=>$licenseId,
          'copyrights'=>$entry['copyrights'],
          'comment'=>$comment));
    }
    $this->heartbeat($filesProceeded - $lastValue);
    return $content;
  }
 
  protected function getVerificationCode(Upload $upload)
  {
    $stmt = __METHOD__;
    $param = array();
    if ($upload->getTreeTableName()=='uploadtree_a') {
      $sql = $upload->getTreeTableName().' WHERE upload_fk=$1 AND';
      $param[] = $upload->getId();
    } else {
      $sql = $upload->getTreeTableName().' WHERE';
      $stmt .= '.'.$upload->getTreeTableName();
    }
 
    $sql = "SELECT STRING_AGG(lower_sha1,'') concat_sha1 FROM
       (SELECT LOWER(pfile_sha1) lower_sha1 FROM pfile, $sql pfile_fk=pfile_pk AND parent IS NOT NULL ORDER BY pfile_sha1) templist";
    $filelistPack = $this->dbManager->getSingleRow($sql,$param,$stmt);
 
    return sha1($filelistPack['concat_sha1']);
  }
 
  protected function getSPDXReportConf($uploadId, $key)
  {
    $sql = "SELECT ri_spdx_selection FROM report_info WHERE upload_fk = $1";
    $getCommentState = $this->dbManager->getSingleRow($sql, array($uploadId), __METHOD__.'.SPDX_license_comment');
    if (!empty($getCommentState['ri_spdx_selection'])) {
      $getCommentStateSingle = explode(',', $getCommentState['ri_spdx_selection']);
      if ($getCommentStateSingle[$key] === "checked") {
        return true;
      }
    }
    return false;
  }
  private function getObligations(int $uploadId, int $groupId): array
  {
    $licenses = $this->licenseClearedGetter->getCleared($uploadId, $this,
      $groupId, true, "license", false);
    $this->heartbeat(0);
    $licensesMain = $this->licenseMainGetter->getCleared($uploadId, $this,
      $groupId, true, null, false);
    $this->heartbeat(0);
    list($obligations, $_) = $this->obligationsGetter->getObligations(
      $licenses['statements'], $licensesMain['statements'], $uploadId,
      $groupId);
    if (empty($obligations)) {
      return [];
    } else {
      return array_column($obligations, "text");
    }
  }
 
  protected function getSPDXDataLicense()
  {
    $dataLic = $this->licenseDao->getLicenseByShortName(self::DATA_LICENSE);
    return $dataLic->getId() . "-" . md5($dataLic->getText());
  }
 
  private function deduplicateLicenseList()
  {
    $localList = array_values($this->licensesInDocument);
    usort($localList,
      function(SpdxLicenseInfo $a, SpdxLicenseInfo $b) {
        return strcmp(
          $a->getLicenseObj()->getSpdxId() . $a->getLicenseObj()->getShortName(),
          $b->getLicenseObj()->getSpdxId() . $b->getLicenseObj()->getShortName());
      }
    );
    for ($i = 0; $i < count($localList) - 1; $i++) {
      if ((! $localList[$i]->isCustomText() && ! $localList[$i + 1]->isCustomText()) &&
          $localList[$i]->getLicenseObj()->getSpdxId() ===
          $localList[$i + 1]->getLicenseObj()->getSpdxId()) {
        $newShortName = $localList[$i + 1]->getLicenseObj()->getShortName();
        if (! StringOperation::stringStartsWith(
            $localList[$i + 1]->getLicenseObj()->getShortName(),
            LicenseRef::SPDXREF_PREFIX)) {
          $newShortName = LicenseRef::SPDXREF_PREFIX .
            $localList[$i + 1]->getLicenseObj()->getShortName();
          $newShortName = preg_replace('/\+$/', '-or-later', $newShortName);
        }
        $md5 = md5($localList[$i + 1]->getLicenseObj()->getText());
        $reportedLicenseShortname = "$newShortName-$md5";
        $licIndex = $localList[$i + 1]->getLicenseObj()->getId() . "-$md5";
        $oldLicObj = $this->licensesInDocument[$licIndex]->getLicenseObj();
        $this->licensesInDocument[$licIndex]->setLicenseObj(
          new License($oldLicObj->getId(), $reportedLicenseShortname,
            $oldLicObj->getFullName(), $oldLicObj->getRisk(),
            $oldLicObj->getText(), $oldLicObj->getUrl(),
            $oldLicObj->getDetectorType(), $oldLicObj->getSpdxId()));
      }
    }
  }
}
 
$agent = new SpdxAgent();
$agent->scheduler_connect();
$agent->run_scheduler_event_loop();
$agent->scheduler_disconnect(0);