cyclonedx.php

Go to the documentation of this file.

<?php
/*
 SPDX-FileCopyrightText: © 2023 Sushant Kumar(sushantmishra02102002@gmail.com)
 
 SPDX-License-Identifier: GPL-2.0-only
*/
namespace Fossology\CycloneDX;
 
use Fossology\Lib\Agent\Agent;
use Fossology\Lib\BusinessRules\LicenseMap;
use Fossology\Lib\Dao\ClearingDao;
use Fossology\Lib\Dao\LicenseDao;
use Fossology\Lib\Dao\TreeDao;
use Fossology\Lib\Dao\UploadDao;
use Fossology\Lib\Data\AgentRef;
use Fossology\Lib\Data\Report\FileNode;
use Fossology\Lib\Data\Report\SpdxLicenseInfo;
use Fossology\Lib\Db\DbManager;
use Fossology\Lib\Data\Package\ComponentType;
use Fossology\Lib\Report\ReportUtils;
 
include_once(__DIR__ . "/version.php");
include_once(__DIR__ . "/reportgenerator.php");
 
class CycloneDXAgent extends Agent
{
  const OUTPUT_FORMAT_KEY = "outputFormat";               
  const DEFAULT_OUTPUT_FORMAT = "cyclonedx_json";                  
  const UPLOADS_ADD_KEY = "uploadsAdd";
 
  private $additionalUploads = [];
 
  private $reportGenerator;
  protected $citations = [];
  private $reportutils;
  private $uploadDao;
  private $clearingDao;
  private $licenseDao;
  protected $dbManager;
  private $licenseMap;
  protected $agentNames = AgentRef::AGENT_LIST;
  protected $uri;
  private $licensesInDocument = [];
  protected $outputFormat = self::DEFAULT_OUTPUT_FORMAT;
  private $packageName;
 
  function __construct()
  {
    // deduce the agent name from the command line arguments
    $args = getopt("", array(
      self::OUTPUT_FORMAT_KEY.'::',
      self::UPLOADS_ADD_KEY.'::'
    ));
    $agentName = "";
    if (array_key_exists(self::OUTPUT_FORMAT_KEY, $args)) {
      $agentName = trim($args[self::OUTPUT_FORMAT_KEY]);
    }
    if (empty($agentName)) {
        $agentName = self::DEFAULT_OUTPUT_FORMAT;
    }
    if (array_key_exists(self::UPLOADS_ADD_KEY, $args)) {
      $uploadsString = $args[self::UPLOADS_ADD_KEY];
      if (!empty($uploadsString)) {
          $this->additionalUploads = explode(',', $uploadsString);
      }
    }
 
    parent::__construct($agentName, AGENT_VERSION, AGENT_REV);
 
    $this->uploadDao = $this->container->get('dao.upload');
    $this->clearingDao = $this->container->get('dao.clearing');
    $this->licenseDao = $this->container->get('dao.license');
    $this->dbManager = $this->container->get('db.manager');
 
    $this->reportutils = new ReportUtils();
    $this->reportGenerator = new BomReportGenerator();
  }
 
  function processUploadId($uploadId)
  {
    $this->licenseMap = new LicenseMap($this->dbManager, $this->groupId, LicenseMap::REPORT, true);
 
    $packageNodes = $this->renderPackage($uploadId);
 
    $this->computeUri($uploadId);
 
    $this->writeReport($packageNodes, $uploadId);
    return true;
  }
 
  protected function getUri($fileBase)
  {
    if (count($this->additionalUploads) > 0) {
      $fileName = $fileBase . "multifile" . "_" . strtoupper($this->outputFormat);
    } else {
      $fileName = $fileBase. strtoupper($this->outputFormat)."_".$this->packageName;
    }
 
    return $fileName .".json" ;
  }
 
  protected function computeUri($uploadId)
  {
    global $SysConf;
    $upload = $this->uploadDao->getUpload($uploadId);
    $this->packageName = $upload->getFilename();
 
    $fileBase = $SysConf['FOSSOLOGY']['path']."/report/";
 
    $this->uri = $this->getUri($fileBase);
  }
 
  protected function renderPackage($uploadId)
  {
    global $SysConf;
    $uploadTreeTableName = $this->uploadDao->getUploadtreeTableName($uploadId);
    $itemTreeBounds = $this->uploadDao->getParentItemBounds($uploadId, $uploadTreeTableName);
    $this->heartbeat(0);
 
    $filesWithLicenses = $this->reportutils
      ->getFilesWithLicensesFromClearings($itemTreeBounds, $this->groupId,
        $this, $this->licensesInDocument);
    $this->heartbeat(0);
 
    $this->reportutils->addClearingStatus($filesWithLicenses, $itemTreeBounds, $this->groupId);
    $this->heartbeat(0);
 
    $this->reportutils->addScannerResults($filesWithLicenses, $itemTreeBounds, $this->groupId, $this->licensesInDocument);
    $this->heartbeat(0);
 
    $this->reportutils->addCopyrightResults($filesWithLicenses, $uploadId);
    $this->heartbeat(0);
 
    $this->citations = [
      'cite-scanner' => [
        'timestamp' => date('c'),
        'attributedTo' => 'tool-fossology-scanners',
        'expressions' => []
      ],
      'cite-analyst' => [
        'timestamp' => date('c'),
        'attributedTo' => 'person-fossology-analyst',
        'expressions' => []
      ]
    ];
 
    $customLicenseTexts = $this->clearingDao->getMainLicenseReportInfos($uploadId, $this->groupId);
 
    $upload = $this->uploadDao->getUpload($uploadId);
    $components = $this->generateFileComponents($filesWithLicenses, $upload->getTreeTableName(), $uploadId, $itemTreeBounds, $customLicenseTexts);
 
    $mainLicenseIds = $this->clearingDao->getMainLicenseIds($uploadId, $this->groupId);
    $mainLicenses = array();
    $seenLicenseIds = array();
    foreach ($mainLicenseIds as $licId) {
      $reportedLicenseId = $this->licenseMap->getProjectedId($licId);
      $mainLicObj = $this->licenseDao->getLicenseById($reportedLicenseId, $this->groupId);
      if ($mainLicObj === null) {
        continue;
      }
 
      $licensedata = $this->getLicenseDataForCycloneDX($mainLicObj, $licId, $customLicenseTexts);
      $licensedata['bom-ref'] = 'lic-clearing-main-' . $licId;
      $licensedata['acknowledgement'] = 'concluded';
      $this->citations['cite-analyst']['expressions'][] = '$..[?(@.bom-ref=="lic-clearing-main-' . $licId . '")]';
      $mainLicenses[] = $this->reportGenerator->createLicense($licensedata);
 
      $customText = array_key_exists($licId, $customLicenseTexts) ? $customLicenseTexts[$licId] : null;
      $licText = !empty($customText) ? $customText : $mainLicObj->getText();
      $reportLicId = $mainLicObj->getId() . "-" . md5($licText);
      $seenLicenseIds[$reportLicId] = true;
    }
 
    foreach ($filesWithLicenses as $fileNode) {
      $isConcluded = !empty($fileNode->getConcludedLicenses());
      $licenseIds = $isConcluded
        ? $fileNode->getConcludedLicenses()
        : $fileNode->getScanners();
      foreach ($licenseIds as $licenseId) {
        if (array_key_exists($licenseId, $this->licensesInDocument) && !array_key_exists($licenseId, $seenLicenseIds)) {
          $seenLicenseIds[$licenseId] = true;
          $licObj = $this->licensesInDocument[$licenseId]->getLicenseObj();
          $isCustomText = $this->licensesInDocument[$licenseId]->isCustomText();
          $licensedata = $this->getLicenseDataForCycloneDX($licObj, $licenseId, $customLicenseTexts, $isCustomText);
          $refPrefix = $isConcluded ? 'clearing' : 'scanner';
          $licensedata['bom-ref'] = 'lic-' . $refPrefix . '-main-' . $licenseId;
          $licensedata['acknowledgement'] = $isConcluded ? 'concluded' : 'declared';
          $this->citations['cite-' . ($isConcluded ? 'analyst' : 'scanner')]['expressions'][] = '$..[?(@.bom-ref=="lic-' . $refPrefix . '-main-' . $licenseId . '")]';
          $mainLicenses[] = $this->reportGenerator->createLicense($licensedata);
        }
      }
    }
 
    $hashes = $this->uploadDao->getUploadHashes($uploadId);
    $serializedhash = array();
    $serializedhash[] = $this->reportGenerator->createHash('SHA-1', $hashes['sha1']);
    $serializedhash[] = $this->reportGenerator->createHash('MD5', $hashes['md5']);
    // Check if sha256 is not empty
    if (array_key_exists('sha256', $hashes) && !empty($hashes['sha256'])) {
      $serializedhash[] = $this->reportGenerator->createHash('SHA-256', $hashes['sha256']);
    }
 
    $allCopyrights = array();
    foreach ($filesWithLicenses as $fileNode) {
      $fileCopyrights = $fileNode->getCopyrights();
      if (!empty($fileCopyrights)) {
        $allCopyrights = array_merge($allCopyrights, $fileCopyrights);
      }
    }
    $allCopyrights = array_unique($allCopyrights);
 
    $reportInfo = $this->uploadDao->getReportInfo($uploadId);
    $componentVersion = ($reportInfo['ri_version'] ?? '');
    if ($componentVersion == 'NA') {
      $componentVersion = '';
    }
    $componentId = ($reportInfo['ri_component_id'] ?? '');
    if ($componentId == 'NA') {
      $componentId = '';
    }
    $componentType = intval($reportInfo['ri_component_type'] ?? 0);
    $generalAssessment = ($reportInfo['ri_general_assesment'] ?? '');
    if ($generalAssessment == 'NA') {
      $generalAssessment = '';
    }
 
    $purl = '';
    $externalReferences = [];
    if (!empty($componentId)) {
      if ($componentType === ComponentType::PURL || $componentType === ComponentType::PACKAGEURL) {
        $purl = $componentId;
      } else {
        $externalReferences[] = [
          'type' => 'distribution',
          'url' => $componentId
        ];
      }
    }
 
    $maincomponentData = array (
      'bomref' => strval($uploadId),
      'type' => 'library',
      'name' => $upload->getFilename(),
      'version' => $componentVersion,
      'hashes' => $serializedhash,
      'scope' => 'required',
      'mimeType' => $this->getMimeType($uploadId),
      'copyright' => implode("\n", $allCopyrights),
      'description' => $generalAssessment,
      'purl' => $purl,
      'externalReferences' => $externalReferences,
      'licenses' => $mainLicenses
    );
    $maincomponent = $this->reportGenerator->createComponent($maincomponentData);
 
    $formattedDate = date('Y-m-d\TH:i:s\Z');
 
    $finalCitations = [];
    if (!empty($this->citations['cite-scanner']['expressions'])) {
      $finalCitations[] = $this->citations['cite-scanner'];
    }
    if (!empty($this->citations['cite-analyst']['expressions'])) {
      $finalCitations[] = $this->citations['cite-analyst'];
    }
 
    $bomdata = array(
      'timestamp' => $formattedDate,
      'tool-version' => $SysConf['BUILD']['VERSION'],
      'maincomponent' => $maincomponent,
      'components' => $components,
      'externalReferences' => $externalReferences,
      'citations' => $finalCitations
    );
 
    return $this->reportGenerator->generateReport($bomdata);
  }
 
  protected function generateFileComponents($filesWithLicenses, $treeTableName, $uploadId, $itemTreeBounds, $customLicenseTexts = array())
  {
    /* @var $treeDao TreeDao */
    $treeDao = $this->container->get('dao.tree');
 
    $stateWoInfos = $this->getCycloneDXReportConf($uploadId, 1);
 
    $filesProceeded = 0;
    $lastValue = 0;
    $components = array();
    foreach ($filesWithLicenses as $fileId => $licenses) {
      $filesProceeded += 1;
      if (($filesProceeded & 2047) == 0) {
        $this->heartbeat($filesProceeded - $lastValue);
        $lastValue = $filesProceeded;
      }
 
      if ($stateWoInfos && empty($licenses->getConcludedLicenses()) &&
          empty($licenses->getScanners()) && empty($licenses->getCopyrights())) {
        continue;
      }
 
      $hashes = $treeDao->getItemHashes($fileId);
      $serializedhash = array();
      $serializedhash[] = $this->reportGenerator->createHash('SHA-1', $hashes['sha1']);
      $serializedhash[] = $this->reportGenerator->createHash('MD5', $hashes['md5']);
      // Check if sha256 is not empty
      if (array_key_exists('sha256', $hashes) && !empty($hashes['sha256'])) {
        $serializedhash[] = $this->reportGenerator->createHash('SHA-256', $hashes['sha256']);
      }
 
      $fileName = $treeDao->getFullPath($fileId, $treeTableName, 0);
      $licensesfound = [];
 
      if (!empty($licenses->getConcludedLicenses())) {
        foreach ($licenses->getConcludedLicenses() as $licenseId) {
          if (array_key_exists($licenseId, $this->licensesInDocument)) {
            $licObj = $this->licensesInDocument[$licenseId]->getLicenseObj();
            $isCustomText = $this->licensesInDocument[$licenseId]->isCustomText();
            $licensedata = $this->getLicenseDataForCycloneDX($licObj, $licenseId, $customLicenseTexts, $isCustomText, $stateOsselot);
            $licensedata['bom-ref'] = 'lic-clearing-' . $licenseId . '-' . $fileId;
            $licensedata['acknowledgement'] = 'concluded';
            $licensesfound[] = $this->reportGenerator->createLicense($licensedata, false);
          }
        }
      } else {
        foreach ($licenses->getScanners() as $licenseId) {
          if (array_key_exists($licenseId, $this->licensesInDocument)) {
            $licObj = $this->licensesInDocument[$licenseId]->getLicenseObj();
            $isCustomText = $this->licensesInDocument[$licenseId]->isCustomText();
            $licensedata = $this->getLicenseDataForCycloneDX($licObj, $licenseId, $customLicenseTexts, $isCustomText, $stateOsselot);
            $licensedata['bom-ref'] = 'lic-scanner-' . $licenseId . '-' . $fileId;
            $licensedata['acknowledgement'] = 'declared';
            $licensesfound[] = $this->reportGenerator->createLicense($licensedata, false);
          }
        }
      }
      if (!empty($fileName)) {
        $mimeType = $this->getFileMimeType($fileId, $treeTableName);
        $componentdata = array(
          'bomref' => $uploadId .'-'. $fileId,
          'type' => 'file',
          'name' => $fileName,
          'hashes' => $serializedhash,
          'mimeType' => $mimeType,
          'copyright' => implode("\n", $licenses->getCopyrights()),
          'licenses' => $licensesfound,
          'acknowledgements' => implode("\n", $licenses->getAcknowledgements()),
          'comments' => implode("\n", $licenses->getComments())
        );
        $components[] = $this->reportGenerator->createComponent($componentdata);
      }
    }
    $this->heartbeat($filesProceeded - $lastValue);
    return $components;
  }
 
  protected function writeReport($packageNodes, $uploadId)
  {
    $fileBase = dirname($this->uri);
 
    if (!is_dir($fileBase)) {
      mkdir($fileBase, 0777, true);
    }
    umask(0133);
 
    $contents = json_encode($packageNodes, JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES);
    // To ensure the file is valid, replace any non-printable characters with a question mark.
    // 'Non-printable' is ASCII < 0x20 (excluding \r, \n and tab) and 0x7F - 0x9F.
    $contents = preg_replace('/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F-\x9F]/u','?',$contents);
    file_put_contents($this->uri, $contents);
    $this->updateReportTable($uploadId, $this->jobId, $this->uri);
  }
 
  protected function updateReportTable($uploadId, $jobId, $fileName)
  {
    $this->reportutils->updateOrInsertReportgenEntry($uploadId, $jobId, $fileName);
  }
 
  private function getLicenseDataForCycloneDX($licObj, $licenseId, $customLicenseTexts, $isCustomText = false, $includeText = true)
  {
    $customText = array_key_exists($licenseId, $customLicenseTexts) ? $customLicenseTexts[$licenseId] : null;
    $licText = !empty($customText) ? $customText : $licObj->getText();
 
    $licensedata = array(
      'url' => $licObj->getUrl()
    );
 
    if (!empty($customText) || $isCustomText) {
      if (!empty($customText)) {
        $prefix = \Fossology\Lib\Data\LicenseRef::SPDXREF_PREFIX;
        $licensedata['name'] = $prefix . $licObj->getShortName() . '-' . md5($customText);
      } else {
        $licensedata['name'] = $licObj->getShortName();
      }
    } else {
      $spdxId = $licObj->getSpdxId();
      if (!empty($spdxId)) {
        $licensedata['id'] = $spdxId;
      } else {
        $licensedata['name'] = $licObj->getFullName();
      }
    }
 
    if ($includeText && !empty($licText)) {
      $licensedata['textContent'] = base64_encode($licText);
      $licensedata['textContentType'] = 'text/plain';
    }
 
    return $licensedata;
  }
 
  protected function getMimeType($uploadId)
  {
    $sql = "SELECT mimetype_name
      FROM upload u
      JOIN pfile pf ON u.pfile_fk = pf.pfile_pk
      JOIN mimetype m ON pf.pfile_mimetypefk = m.mimetype_pk
      WHERE u.upload_pk = $1";
 
    $row = $this->dbManager->getSingleRow($sql, [$uploadId], __METHOD__);
    return $row['mimetype_name'];
  }
 
  protected function getFileMimeType($fileId, $treeTableName)
  {
    $sql = "SELECT m.mimetype_name
      FROM $treeTableName ut
      JOIN pfile pf ON ut.pfile_fk = pf.pfile_pk
      LEFT JOIN mimetype m ON pf.pfile_mimetypefk = m.mimetype_pk
      WHERE ut.uploadtree_pk = $1";
 
    $row = $this->dbManager->getSingleRow($sql, [$fileId], __METHOD__);
    return $row['mimetype_name'] ?? 'application/octet-stream';
  }
 
  protected function getCycloneDXReportConf($uploadId, $key)
  {
    $settings = $this->uploadDao->getCyclonedxSettings($uploadId);
    if (!empty($settings)) {
      $settingsArr = explode(',', $settings);
      if (isset($settingsArr[$key]) && $settingsArr[$key] === "checked") {
        return true;
      }
    }
    return false;
  }
}
 
$agent = new CycloneDXAgent();
$agent->scheduler_connect();
$agent->run_scheduler_event_loop();
$agent->scheduler_disconnect(0);