FOSSology  4.5.1
Open Source License Compliance by Open Source Software
spdx.php
Go to the documentation of this file.
1 <?php
2 /*
3  SPDX-FileCopyrightText: © 2015-2016,2023 Siemens AG
4  SPDX-FileCopyrightText: © 2017 TNG Technology Consulting GmbH
5 
6  SPDX-License-Identifier: GPL-2.0-only
7 */
43 namespace Fossology\Spdx;
44 
64 use Twig\Environment;
65 
66 include_once(__DIR__ . "/spdxutils.php");
67 
68 include_once(__DIR__ . "/version.php");
69 include_once(__DIR__ . "/services.php");
70 
75 class SpdxAgent extends Agent
76 {
77 
78  const OUTPUT_FORMAT_KEY = "outputFormat";
79  const DEFAULT_OUTPUT_FORMAT = "spdx2";
80  const AVAILABLE_OUTPUT_FORMATS = "spdx2,spdx2tv,dep5,spdx2csv,spdx3jsonld,spdx3json,spdx3rdf,spdx3tv";
81  const UPLOAD_ADDS = "uploadsAdd";
82  const DATA_LICENSE = "CC0-1.0";
83 
87  private $uploadDao;
91  private $clearingDao;
95  private $licenseDao;
99  protected $dbManager;
103  protected $renderer;
107  private $licenseMap;
127  private $reportutils;
131  protected $agentNames = AgentRef::AGENT_LIST;
135  protected $filebasename = null;
139  protected $uri;
143  protected $filename;
148  private $licensesInDocument = [];
161 
162  function __construct()
163  {
164  // deduce the agent name from the command line arguments
165  $args = getopt("", array(self::OUTPUT_FORMAT_KEY.'::'));
166  $agentName = "";
167  if (array_key_exists(self::OUTPUT_FORMAT_KEY, $args)) {
168  $agentName = trim($args[self::OUTPUT_FORMAT_KEY]);
169  }
170  if (empty($agentName)) {
172  }
173 
174  parent::__construct($agentName, AGENT_VERSION, AGENT_REV);
175 
176  $this->uploadDao = $this->container->get('dao.upload');
177  $this->clearingDao = $this->container->get('dao.clearing');
178  $this->licenseDao = $this->container->get('dao.license');
179  $this->dbManager = $this->container->get('db.manager');
180  $this->renderer = $this->container->get('twig.environment');
181  $this->renderer->setCache(false);
182 
183  $this->agentSpecifLongOptions[] = self::UPLOAD_ADDS.':';
184  $this->agentSpecifLongOptions[] = self::OUTPUT_FORMAT_KEY.':';
185 
186  $this->licenseClearedGetter = new LicenseClearedGetter();
187  $this->licenseMainGetter = new LicenseMainGetter();
188  $this->obligationsGetter = new ObligationsGetter();
189  $this->reportutils = new ReportUtils();
190  }
191 
197  protected function preWorkOnArgs($args)
198  {
199  if ((!array_key_exists(self::OUTPUT_FORMAT_KEY, $args)
200  || $args[self::OUTPUT_FORMAT_KEY] === "")
201  && array_key_exists(self::UPLOAD_ADDS,$args)) {
202  $args = SpdxUtils::preWorkOnArgsFlp($args,self::UPLOAD_ADDS,self::OUTPUT_FORMAT_KEY);
203  } else {
204  if (!array_key_exists(self::UPLOAD_ADDS,$args) || $args[self::UPLOAD_ADDS] === "") {
205  $args = SpdxUtils::preWorkOnArgsFlp($args,self::UPLOAD_ADDS,self::OUTPUT_FORMAT_KEY);
206  }
207  }
208  return $args;
209  }
210 
215  function processUploadId($uploadId)
216  {
217  $args = $this->preWorkOnArgs($this->args);
218 
219  if (array_key_exists(self::OUTPUT_FORMAT_KEY,$args)) {
220  $possibleOutputFormat = trim($args[self::OUTPUT_FORMAT_KEY]);
221  if (in_array($possibleOutputFormat, explode(',',self::AVAILABLE_OUTPUT_FORMATS))) {
222  $this->outputFormat = $possibleOutputFormat;
223  }
224  }
225  $this->licenseMap = new LicenseMap($this->dbManager, $this->groupId, LicenseMap::REPORT, true);
226  $this->computeUri($uploadId);
227  $docLicense = $this->licenseDao->getLicenseByShortName(self::DATA_LICENSE);
228  $docLicenseId = $docLicense->getId() . "-" . md5($docLicense->getText());
229  $this->licensesInDocument[$docLicenseId] = (new SpdxLicenseInfo())
230  ->setLicenseObj($docLicense)
231  ->setListedLicense(true)
232  ->setCustomText(false)
233  ->setTextPrinted(true);
234  $packageNodes = $this->renderPackage($uploadId);
235  $additionalUploadIds = array_key_exists(self::UPLOAD_ADDS,$args) ? explode(',',$args[self::UPLOAD_ADDS]) : array();
236  $packageIds = array($uploadId);
237  foreach ($additionalUploadIds as $additionalId) {
238  $packageNodes .= $this->renderPackage($additionalId);
239  $packageIds[] = $additionalId;
240  }
241  $this->writeReport($packageNodes, $packageIds, $uploadId);
242  return true;
243  }
244 
250  protected function getTemplateFile($partname)
251  {
252  $prefix = $this->outputFormat . "-";
253  $postfix = ".twig";
254  switch ($this->outputFormat) {
255  case "spdx2":
256  $postfix = ".xml" . $postfix;
257  break;
258  case "spdx2csv":
259  case "spdx2tv":
260  break;
261  case "dep5":
262  $prefix .= "copyright-";
263  break;
264  case "spdx3jsonld":
265  break;
266  case "spdx3rdf":
267  $postfix = ".xml" . $postfix;
268  break;
269  case "spdx3json":
270  break;
271  case "spdx3tv":
272  break;
273  }
274  return $prefix . $partname . $postfix;
275  }
276 
284  protected function getFileBasename($packageName)
285  {
286  if ($this->filebasename == null) {
287  $fileName = strtoupper($this->outputFormat)."_".$packageName;
288  switch ($this->outputFormat) {
289  case "spdx2":
290  $fileName .= ".spdx.rdf";
291  break;
292  case "spdx2tv":
293  $fileName .= ".spdx";
294  break;
295  case "spdx2csv":
296  $fileName .= ".csv";
297  break;
298  case "dep5":
299  $fileName .= ".txt";
300  break;
301  case "spdx3jsonld":
302  $fileName = $fileName .".jsonld";
303  break;
304  case "spdx3json":
305  $fileName = $fileName .".json";
306  break;
307  case "spdx3rdf":
308  $fileName = $fileName .".spdx.rdf";
309  break;
310  case "spdx3tv":
311  $fileName = $fileName .".spdx";
312  break;
313  }
314  $this->filebasename = $fileName;
315  }
316  return $this->filebasename;
317  }
318 
324  protected function getFileName($packageName)
325  {
326  global $SysConf;
327  $fileBase = $SysConf['FOSSOLOGY']['path']."/report/";
328  return $fileBase. $this->getFileBasename($packageName);
329  }
330 
336  protected function getUri($packageName)
337  {
338  global $SysConf;
339  $url=$SysConf['SYSCONFIG']['FOSSologyURL'];
340  if (substr( $url, 0, 4 ) !== "http") {
341  $url="http://".$url;
342  }
343 
344  return rtrim($url, '/') . '/' . $this->getFileBasename($packageName);
345  }
346 
352  protected function renderPackage($uploadId)
353  {
354  $uploadTreeTableName = $this->uploadDao->getUploadtreeTableName($uploadId);
355  $itemTreeBounds = $this->uploadDao->getParentItemBounds($uploadId,$uploadTreeTableName);
356  $this->heartbeat(0);
357 
358  $filesWithLicenses = $this->reportutils
359  ->getFilesWithLicensesFromClearings($itemTreeBounds, $this->groupId,
360  $this, $this->licensesInDocument);
361  $this->heartbeat(0);
362 
363  $this->reportutils->addClearingStatus($filesWithLicenses,$itemTreeBounds, $this->groupId);
364  $this->heartbeat(0);
365 
366  $scannerIDs = $this->reportutils->addScannerResults($filesWithLicenses, $itemTreeBounds, $this->groupId, $this->licensesInDocument);
367  $licenseComment = "";
368  if (!empty($scannerIDs)) {
369  $licenseComment = $this->getLicenseComment($scannerIDs);
370  }
371  $this->heartbeat(0);
372 
373  $this->reportutils->addCopyrightResults($filesWithLicenses, $uploadId);
374  $this->heartbeat(0);
375 
376  $upload = $this->uploadDao->getUpload($uploadId);
377  $fileNodes = $this->generateFileNodes($filesWithLicenses, $upload->getTreeTableName(), $uploadId);
378 
379  $mainLicenseIds = $this->clearingDao->getMainLicenseIds($uploadId, $this->groupId);
380  $mainLicenses = array();
381  foreach ($mainLicenseIds as $licId) {
382  $reportedLicenseId = $this->licenseMap->getProjectedId($licId);
383  $mainLicense = $this->licenseDao->getLicenseById($reportedLicenseId, $this->groupId);
384  if ($mainLicense === null) {
385  error_log(
386  "spdx: Error: main license ID {$reportedLicenseId} not found; skipping."
387  );
388  exit;
389  }
390  $reportLicId = $mainLicense->getId() . "-" . md5($mainLicense->getText());
391  $mainLicenses[] = $reportLicId;
392  if (!array_key_exists($reportLicId, $this->licensesInDocument)) {
393  $listedLicense = stripos($mainLicense->getSpdxId(),
394  LicenseRef::SPDXREF_PREFIX) !== 0;
395  $this->licensesInDocument[$reportLicId] = (new SpdxLicenseInfo())
396  ->setLicenseObj($mainLicense)
397  ->setCustomText(false)
398  ->setListedLicense($listedLicense);
399  }
400  }
401  $mainLicenseString = [];
402  if ($this->outputFormat == "spdx2tv" ||
403  $this->outputFormat == "spdx2csv" ||
404  $this->outputFormat == "spdx3tv") {
405  foreach ($mainLicenses as $mainLicense) {
406  $shortName = $this->licensesInDocument[$mainLicense]
407  ->getLicenseObj()->getShortName();
408  if (StringOperation::stringStartsWith($shortName,
409  LicenseRef::SPDXREF_PREFIX)) {
410  $mainLicenseString[] = $shortName;
411  } else {
412  $mainLicenseString[] = $this->licensesInDocument[$mainLicense]
413  ->getLicenseObj()->getSpdxId();
414  }
415  }
416  $mainLicenseString = SpdxUtils::implodeLicenses(
417  SpdxUtils::removeEmptyLicenses($mainLicenseString));
418  }
419 
420  $hashes = $this->uploadDao->getUploadHashes($uploadId);
421 
422  $reportInfo = $this->uploadDao->getReportInfo($uploadId);
423  $componentId = $reportInfo['ri_component_id'];
424  $componentType = $reportInfo['ri_component_type'];
425  $componentVersion = $reportInfo['ri_version'];
426  $generalAssessment = $reportInfo['ri_general_assesment'];
427  $releaseDate = $reportInfo['ri_release_date'];
428  if ($componentId == "NA") {
429  $componentId = "";
430  }
431  if ($componentVersion == "NA") {
432  $componentVersion = "";
433  }
434  if ($generalAssessment == "NA") {
435  $generalAssessment = "";
436  }
437  if ($releaseDate == "NA") {
438  $releaseDate = "";
439  } else {
440  $timeStamp = strtotime($releaseDate);
441  if ($timeStamp != -1) {
442  $releaseDate = date("Y-m-d\\T00:00:00\\Z", $timeStamp);
443  } else {
444  $releaseDate = "";
445  }
446  }
447  if ($componentType == ComponentType::MAVEN) {
448  $componentType = "maven-central";
449  } elseif ($componentType == ComponentType::PACKAGEURL) {
450  $componentType = "purl";
451  } else {
452  if (!empty($componentType)) {
453  $componentType = ComponentType::TYPE_MAP[$componentType];
454  } else {
456  }
457  }
458  $obligations = $this->getObligations($uploadId, $this->groupId);
459 
460  return $this->renderString($this->getTemplateFile('package'), [
461  'packageId' => $uploadId,
462  'uri' => $this->uri,
463  'packageName' => $upload->getFilename(),
464  'packageVersion' => $componentVersion,
465  'releaseDate' => $releaseDate,
466  'generalAssessment' => $generalAssessment,
467  'uploadName' => $upload->getFilename(),
468  'componentType' => $componentType,
469  'componentId' => htmlspecialchars($componentId),
470  'sha1' => $hashes['sha1'],
471  'md5' => $hashes['md5'],
472  'sha256' => $hashes['sha256'],
473  'verificationCode' => $this->getVerificationCode($upload),
474  'mainLicenses' => $mainLicenses,
475  'mainLicenseString' => $mainLicenseString,
476  'licenseComments' => $licenseComment,
477  'fileNodes' => $fileNodes,
478  'obligations' => $obligations,
479  'licenseList' => $this->licensesInDocument
480  ]);
481  }
482 
487  protected function getLicenseComment($scannerIds)
488  {
490  $func = function($scannerId) use ($agentDao)
491  {
492  return $agentDao->getAgentName($scannerId)." (".$agentDao->getAgentRev($scannerId).")";
493  };
494  $scannerNames = array_map($func, $scannerIds);
495  return "licenseInfoInFile determined by Scanners:\n - ".implode("\n - ",$scannerNames);
496  }
497 
506  protected function toLicensesWithFilesAdder(&$filesWithLicenses, $licenses, $copyrights, $file, $fullPath)
507  {
508  if (!array_key_exists($licenses, $filesWithLicenses)) {
509  $filesWithLicenses[$licenses]['files']=array();
510  $filesWithLicenses[$licenses]['copyrights']=array();
511  }
512  if (empty($copyrights)) {
513  $copyrights = array();
514  }
515  $filesWithLicenses[$licenses]['files'][$file] = $fullPath;
516  foreach ($copyrights as $copyright) {
517  if (!in_array($copyright, $filesWithLicenses[$licenses]['copyrights'])) {
518  $filesWithLicenses[$licenses]['copyrights'][] = $copyright;
519  }
520  }
521  }
522 
529  protected function toLicensesWithFiles(&$filesWithLicenses, $treeTableName)
530  {
531  $licensesWithFiles = array();
532  $treeDao = $this->container->get('dao.tree');
533  $filesProceeded = 0;
534  foreach ($filesWithLicenses as $fileId => $fileNode) {
535  $filesProceeded += 1;
536  if (($filesProceeded&2047)==0) {
537  $this->heartbeat(0);
538  }
539  $fullPath = $treeDao->getFullPath($fileId, $treeTableName, 0);
540  if (! empty($fileNode->getConcludedLicenses())) {
541  $licenses = [];
542  foreach ($fileNode->getConcludedLicenses() as $license) {
543  $licenses[] = $this->licensesInDocument[$license]
544  ->getLicenseObj()->getSpdxId();
545  }
546  $licenses = SpdxUtils::implodeLicenses(
547  SpdxUtils::removeEmptyLicenses(array_unique($licenses)));
548  $this->toLicensesWithFilesAdder($licensesWithFiles,
549  $licenses, $fileNode->getCopyrights(), $fileId, $fullPath);
550  } else {
551  if (! empty($fileNode->getScanners())) {
552  $implodedLicenses = [];
553  foreach ($fileNode->getScanners() as $license) {
554  $implodedLicenses[] = $this->licensesInDocument[$license]
555  ->getLicenseObj()->getSpdxId();
556  }
557  $implodedLicenses = SpdxUtils::implodeLicenses(
558  SpdxUtils::removeEmptyLicenses(array_unique($implodedLicenses)));
559  if ($fileNode->isCleared()) {
560  $msgLicense = "None (scanners found: " . $implodedLicenses . ")";
561  } else {
562  $msgLicense = "NoLicenseConcluded (scanners found: " . $implodedLicenses . ")";
563  }
564  } else {
565  if ($fileNode->isCleared()) {
566  $msgLicense = "None";
567  } else {
568  $msgLicense = "NoLicenseConcluded";
569  }
570  }
571  $this->toLicensesWithFilesAdder($licensesWithFiles, $msgLicense,
572  $fileNode->getCopyrights(), $fileId, $fullPath);
573  }
574  }
575  return $licensesWithFiles;
576  }
577 
582  protected function computeUri($uploadId)
583  {
584  $upload = $this->uploadDao->getUpload($uploadId);
585  $packageName = $upload->getFilename();
586 
587  $this->uri = $this->getUri($packageName);
588  $this->filename = $this->getFileName($packageName);
589  }
590 
597  protected function writeReport(&$packageNodes, $packageIds, $uploadId)
598  {
599  global $SysConf;
600 
601  $fileBase = dirname($this->filename);
602 
603  if (!is_dir($fileBase)) {
604  mkdir($fileBase, 0777, true);
605  }
606  umask(0133);
607 
608  $organizationName = $SysConf['SYSCONFIG']["ReportHeaderText"] ?? 'FOSSology';
609  $version = $SysConf['BUILD']['VERSION'];
610 
611  $uploadTreeTableName = $this->uploadDao->getUploadtreeTableName($uploadId);
612  $itemTreeBounds = $this->uploadDao->getParentItemBounds($uploadId,$uploadTreeTableName);
613  $this->heartbeat(0);
614 
615  $filesWithLicenses = $this->reportutils
616  ->getFilesWithLicensesFromClearings($itemTreeBounds, $this->groupId,
617  $this, $this->licensesInDocument);
618  $this->heartbeat(0);
619  $upload = $this->uploadDao->getUpload($uploadId);
620  $this->generateFileNodes($filesWithLicenses, $upload->getTreeTableName(), $uploadId);
621  $this->declaredLicenseFileIds = array_unique(array_diff($this->declaredLicenseFileIds, $this->concludedLicenseFileIds));
622  $this->concludedLicenseFileIds = array_unique($this->concludedLicenseFileIds);
623  $message = $this->renderString($this->getTemplateFile('document'),array(
624  'documentName' => $fileBase,
625  'uri' => $this->uri,
626  'userName' => $this->container->get('dao.user')->getUserName($this->userId) . " (" . $this->container->get('dao.user')->getUserEmail($this->userId) . ")",
627  'organisation' => $organizationName,
628  'concludedLicenseFileIds'=>$this->concludedLicenseFileIds,
629  'declaredLicenseFileIds'=>$this->declaredLicenseFileIds,
630  'toolVersion' => 'fossology-' . $version,
631  'packageNodes' => $packageNodes,
632  'packageIds' => $packageIds,
633  'dataLicense' => $this->getSPDXDataLicense(),
634  'licenseList' => $this->licensesInDocument
635  )
636  );
637 
638  // To ensure the file is valid, replace any non-printable characters with a question mark.
639  // 'Non-printable' is ASCII < 0x20 (excluding \r, \n and tab) and 0x7F (delete).
640  $message = preg_replace('/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/','?',$message);
641 
642  file_put_contents($this->filename, $message);
643  $this->updateReportTable($uploadId, $this->jobId, $this->filename);
644  }
645 
652  protected function updateReportTable($uploadId, $jobId, $fileName)
653  {
654  $this->reportutils->updateOrInsertReportgenEntry($uploadId, $jobId, $fileName);
655  }
656 
663  protected function renderString($templateName, $vars)
664  {
665  return $this->renderer->load($templateName)->render($vars);
666  }
667 
675  protected function generateFileNodes($filesWithLicenses, $treeTableName, $uploadId)
676  {
677  $this->deduplicateLicenseList();
678  if (strcmp($this->outputFormat, "dep5") !== 0) {
679  return $this->generateFileNodesByFiles($filesWithLicenses, $treeTableName, $uploadId);
680  } else {
681  return $this->generateFileNodesByLicenses($filesWithLicenses, $treeTableName);
682  }
683  }
684 
692  protected function generateFileNodesByFiles($filesWithLicenses, $treeTableName, $uploadId)
693  {
694  /* @var $treeDao TreeDao */
695  $treeDao = $this->container->get('dao.tree');
696  $filesProceeded = 0;
697  $lastValue = 0;
698  $content = '';
699  $textToBePrinted = [];
700  foreach ($filesWithLicenses as $fileId => $fileData) {
701  $filesProceeded += 1;
702  if (($filesProceeded & 2047) == 0) {
703  $this->heartbeat($filesProceeded - $lastValue);
704  $lastValue = $filesProceeded;
705  }
706  $hashes = $treeDao->getItemHashes($fileId);
707  $fileName = $treeDao->getFullPath($fileId, $treeTableName, 0);
708  $stateComment = $this->getSPDXReportConf($uploadId, 0);
709  $stateWoInfos = $this->getSPDXReportConf($uploadId, 1);
710  foreach ($fileData->getConcludedLicenses() as $license) {
711  $this->concludedLicenseFileIds[] = $fileId;
712  if (! $this->licensesInDocument[$license]->isTextPrinted()) {
713  $textToBePrinted[] = $license;
714  }
715  }
716  foreach ($fileData->getScanners() as $license) {
717  $this->declaredLicenseFileIds[] = $fileId;
718  if (! $this->licensesInDocument[$license]->isTextPrinted()) {
719  $textToBePrinted[] = $license;
720  }
721  }
722  $concludedLicensesString = [];
723  if ($this->outputFormat == "spdx2tv" ||
724  $this->outputFormat == "spdx2csv" ||
725  $this->outputFormat == "spdx3tv") {
726  foreach ($fileData->getConcludedLicenses() as $license) {
727  $shortName = $this->licensesInDocument[$license]
728  ->getLicenseObj()->getShortName();
729  if (StringOperation::stringStartsWith($shortName,
730  LicenseRef::SPDXREF_PREFIX)) {
731  $concludedLicensesString[] = $shortName;
732  } else {
733  $concludedLicensesString[] = $this->licensesInDocument[$license]
734  ->getLicenseObj()->getSpdxId();
735  }
736  }
737  $concludedLicensesString = SpdxUtils::implodeLicenses(
738  SpdxUtils::removeEmptyLicenses($concludedLicensesString));
739  }
740  if (!$stateWoInfos ||
741  ($stateWoInfos && (!empty($fileData->getConcludedLicenses()) ||
742  !empty($fileData->getScanners()) || !empty($fileData->getCopyrights())))) {
743  $fileData->setAcknowledgements(
744  SpdxUtils::cleanTextArray($fileData->getAcknowledgements()));
745  $fileData->setComments(
746  SpdxUtils::cleanTextArray($fileData->getComments()));
747  $dataTemplate = array(
748  'fileId' => $fileId,
749  'sha1' => $hashes['sha1'],
750  'md5' => $hashes['md5'],
751  'sha256' => $hashes['sha256'],
752  'uri' => $this->uri,
753  'fileName' => $fileName,
754  'fileDirName' => dirname($fileName),
755  'fileBaseName' => basename($fileName),
756  'fileData' => $fileData,
757  'licenseList' => $this->licensesInDocument,
758  'concludedLicensesString' => $concludedLicensesString,
759  'licenseCommentState' => $stateComment
760  );
761  $content .= $this->renderString($this->getTemplateFile('file'),
762  $dataTemplate);
763  }
764  foreach ($textToBePrinted as $license) {
765  $this->licensesInDocument[$license]->setTextPrinted(true);
766  }
767  }
768  $this->heartbeat($filesProceeded - $lastValue);
769  return $content;
770  }
771 
778  protected function generateFileNodesByLicenses($filesWithLicenses, $treeTableName)
779  {
780  $licensesWithFiles = $this->toLicensesWithFiles($filesWithLicenses, $treeTableName);
781 
782  $content = '';
783  $filesProceeded = 0;
784  $lastStep = 0;
785  $lastValue = 0;
786  foreach ($licensesWithFiles as $licenseId=>$entry) {
787  $filesProceeded += count($entry['files']);
788  if ($filesProceeded&(~2047) > $lastStep) {
789  $this->heartbeat($filesProceeded - $lastValue);
790  $lastStep = $filesProceeded&(~2047) + 2048;
791  $lastValue = $filesProceeded;
792  }
793 
794  $comment = "";
795  if (strrpos($licenseId, "NoLicenseConcluded (scanners found: ", -strlen($licenseId)) !== false) {
796  $comment = substr($licenseId,20,strlen($licenseId)-21);
797  $licenseId = "NoLicenseConcluded";
798  } elseif (strrpos($licenseId, "None (scanners found: ", -strlen($licenseId)) !== false) {
799  $comment = substr($licenseId,6,strlen($licenseId)-7);
800  $licenseId = "None";
801  }
802 
803  $content .= $this->renderString($this->getTemplateFile('file'),array(
804  'fileNames'=>$entry['files'],
805  'license'=>$licenseId,
806  'copyrights'=>$entry['copyrights'],
807  'comment'=>$comment));
808  }
809  $this->heartbeat($filesProceeded - $lastValue);
810  return $content;
811  }
812 
821  protected function getVerificationCode(Upload $upload)
822  {
823  $stmt = __METHOD__;
824  $param = array();
825  if ($upload->getTreeTableName()=='uploadtree_a') {
826  $sql = $upload->getTreeTableName().' WHERE upload_fk=$1 AND';
827  $param[] = $upload->getId();
828  } else {
829  $sql = $upload->getTreeTableName().' WHERE';
830  $stmt .= '.'.$upload->getTreeTableName();
831  }
832 
833  $sql = "SELECT STRING_AGG(lower_sha1,'') concat_sha1 FROM
834  (SELECT LOWER(pfile_sha1) lower_sha1 FROM pfile, $sql pfile_fk=pfile_pk AND parent IS NOT NULL ORDER BY pfile_sha1) templist";
835  $filelistPack = $this->dbManager->getSingleRow($sql,$param,$stmt);
836 
837  return sha1($filelistPack['concat_sha1']);
838  }
839 
846  protected function getSPDXReportConf($uploadId, $key)
847  {
848  $sql = "SELECT ri_spdx_selection FROM report_info WHERE upload_fk = $1";
849  $getCommentState = $this->dbManager->getSingleRow($sql, array($uploadId), __METHOD__.'.SPDX_license_comment');
850  if (!empty($getCommentState['ri_spdx_selection'])) {
851  $getCommentStateSingle = explode(',', $getCommentState['ri_spdx_selection']);
852  if ($getCommentStateSingle[$key] === "checked") {
853  return true;
854  }
855  }
856  return false;
857  }
858 
865  private function getObligations(int $uploadId, int $groupId): array
866  {
867  $licenses = $this->licenseClearedGetter->getCleared($uploadId, $this,
868  $groupId, true, "license", false);
869  $this->heartbeat(0);
870  $licensesMain = $this->licenseMainGetter->getCleared($uploadId, $this,
871  $groupId, true, null, false);
872  $this->heartbeat(0);
873  list($obligations, $_) = $this->obligationsGetter->getObligations(
874  $licenses['statements'], $licensesMain['statements'], $uploadId,
875  $groupId);
876  if (empty($obligations)) {
877  return [];
878  } else {
879  return array_column($obligations, "text");
880  }
881  }
882 
887  protected function getSPDXDataLicense()
888  {
889  $dataLic = $this->licenseDao->getLicenseByShortName(self::DATA_LICENSE);
890  return $dataLic->getId() . "-" . md5($dataLic->getText());
891  }
892 
904  private function deduplicateLicenseList()
905  {
906  $localList = array_values($this->licensesInDocument);
907  usort($localList,
908  function(SpdxLicenseInfo $a, SpdxLicenseInfo $b) {
909  return strcmp(
910  $a->getLicenseObj()->getSpdxId() . $a->getLicenseObj()->getShortName(),
911  $b->getLicenseObj()->getSpdxId() . $b->getLicenseObj()->getShortName());
912  }
913  );
914  for ($i = 0; $i < count($localList) - 1; $i++) {
915  if ((! $localList[$i]->isCustomText() && ! $localList[$i + 1]->isCustomText()) &&
916  $localList[$i]->getLicenseObj()->getSpdxId() ===
917  $localList[$i + 1]->getLicenseObj()->getSpdxId()) {
918  $newShortName = $localList[$i + 1]->getLicenseObj()->getShortName();
920  $localList[$i + 1]->getLicenseObj()->getShortName(),
921  LicenseRef::SPDXREF_PREFIX)) {
922  $newShortName = LicenseRef::SPDXREF_PREFIX .
923  $localList[$i + 1]->getLicenseObj()->getShortName();
924  $newShortName = preg_replace('/\+$/', '-or-later', $newShortName);
925  }
926  $md5 = md5($localList[$i + 1]->getLicenseObj()->getText());
927  $reportedLicenseShortname = "$newShortName-$md5";
928  $licIndex = $localList[$i + 1]->getLicenseObj()->getId() . "-$md5";
929  $oldLicObj = $this->licensesInDocument[$licIndex]->getLicenseObj();
930  $this->licensesInDocument[$licIndex]->setLicenseObj(
931  new License($oldLicObj->getId(), $reportedLicenseShortname,
932  $oldLicObj->getFullName(), $oldLicObj->getRisk(),
933  $oldLicObj->getText(), $oldLicObj->getUrl(),
934  $oldLicObj->getDetectorType(), $oldLicObj->getSpdxId()));
935  }
936  }
937  }
938 }
939 
940 $agent = new SpdxAgent();
941 $agent->scheduler_connect();
942 $agent->run_scheduler_event_loop();
943 $agent->scheduler_disconnect(0);
Structure of an Agent with all required parameters.
Definition: Agent.php:41
heartbeat($newProcessed)
Send hear beat to the scheduler.
Definition: Agent.php:203
Wrapper class for license map.
Definition: LicenseMap.php:19
static stringStartsWith($haystack, $needle)
SPDX2 agent.
Definition: spdx.php:76
const DEFAULT_OUTPUT_FORMAT
Default output format.
Definition: spdx.php:79
getLicenseComment($scannerIds)
Definition: spdx.php:487
getFileName($packageName)
Get absolute path for report.
Definition: spdx.php:324
getVerificationCode(Upload $upload)
Get a unique identifier for a given upload.
Definition: spdx.php:821
getTemplateFile($partname)
Get TWIG template file based on output format.
Definition: spdx.php:250
const UPLOAD_ADDS
Argument for additional uploads.
Definition: spdx.php:81
getObligations(int $uploadId, int $groupId)
Definition: spdx.php:865
processUploadId($uploadId)
Given an upload ID, process the items in it.
Definition: spdx.php:215
const DATA_LICENSE
Data license for SPDX reports.
Definition: spdx.php:82
updateReportTable($uploadId, $jobId, $fileName)
Update the reportgen table with new report path.
Definition: spdx.php:652
preWorkOnArgs($args)
Parse arguments.
Definition: spdx.php:197
renderString($templateName, $vars)
Render a twig template.
Definition: spdx.php:663
const AVAILABLE_OUTPUT_FORMATS
Output formats available.
Definition: spdx.php:80
renderPackage($uploadId)
Given an upload id, render the report string.
Definition: spdx.php:352
getFileBasename($packageName)
Generate report basename based on upload name.
Definition: spdx.php:284
generateFileNodesByLicenses($filesWithLicenses, $treeTableName)
For each file, generate the nodes by licenses.
Definition: spdx.php:778
toLicensesWithFiles(&$filesWithLicenses, $treeTableName)
Map findings to the files.
Definition: spdx.php:529
generateFileNodes($filesWithLicenses, $treeTableName, $uploadId)
Generate report nodes for files.
Definition: spdx.php:675
getSPDXReportConf($uploadId, $key)
Get spdx license comment state for a given upload.
Definition: spdx.php:846
toLicensesWithFilesAdder(&$filesWithLicenses, $licenses, $copyrights, $file, $fullPath)
Map licenses, copyrights, files and full path to filesWithLicenses array.
Definition: spdx.php:506
generateFileNodesByFiles($filesWithLicenses, $treeTableName, $uploadId)
For each file, generate the nodes by files.
Definition: spdx.php:692
writeReport(&$packageNodes, $packageIds, $uploadId)
Write the report the file and update report table.
Definition: spdx.php:597
computeUri($uploadId)
For a given upload, compute the URI and filename for the report.
Definition: spdx.php:582
const OUTPUT_FORMAT_KEY
Argument key for output format.
Definition: spdx.php:78
getUri($packageName)
Get the URI for the given package.
Definition: spdx.php:336
deduplicateLicenseList()
De-duplicate license list by comparing licenses with the same SPDX ID.
Definition: spdx.php:904
static preWorkOnArgsFlp($args, $key1, $key2)
For a given set of arguments assign $args[$key1] and $args[$key2].
Definition: spdxutils.php:28
static implodeLicenses($licenses)
Implode licenses with "AND" or "OR".
Definition: spdxutils.php:96
static removeEmptyLicenses($licenses)
Definition: spdxutils.php:147
static cleanTextArray($texts)
Definition: spdxutils.php:123
char * trim(char *ptext)
Trimming whitespace.
Definition: fossconfig.c:690
int jobId
The id of the job.
fo_dbManager * dbManager
fo_dbManager object
Definition: process.c:16
FUNCTION char * strtoupper(char *s)
Helper function to upper case a string.
Definition: utils.c:39
Namespace used by SPDX2 agent.
list_t type structure used to keep various lists. (e.g. there are multiple lists).
Definition: nomos.h:308