67 include_once(__DIR__ .
"/spdxutils.php");
69 include_once(__DIR__ .
"/version.php");
70 include_once(__DIR__ .
"/services.php");
163 function __construct()
166 $args = getopt(
"", array(self::OUTPUT_FORMAT_KEY.
'::'));
168 if (array_key_exists(self::OUTPUT_FORMAT_KEY,
$args)) {
175 parent::__construct(
$agentName, AGENT_VERSION, AGENT_REV);
177 $this->uploadDao = $this->container->get(
'dao.upload');
178 $this->clearingDao = $this->container->get(
'dao.clearing');
179 $this->licenseDao = $this->container->get(
'dao.license');
180 $this->
dbManager = $this->container->get(
'db.manager');
181 $this->renderer = $this->container->get(
'twig.environment');
182 $this->renderer->setCache(
false);
184 $this->agentSpecifLongOptions[] = self::UPLOAD_ADDS.
':';
185 $this->agentSpecifLongOptions[] = self::OUTPUT_FORMAT_KEY.
':';
200 if ((!array_key_exists(self::OUTPUT_FORMAT_KEY,
$args)
201 ||
$args[self::OUTPUT_FORMAT_KEY] ===
"")
202 && array_key_exists(self::UPLOAD_ADDS,
$args)) {
205 if (!array_key_exists(self::UPLOAD_ADDS,
$args) ||
$args[self::UPLOAD_ADDS] ===
"") {
220 if (array_key_exists(self::OUTPUT_FORMAT_KEY,
$args)) {
221 $possibleOutputFormat =
trim(
$args[self::OUTPUT_FORMAT_KEY]);
222 if (in_array($possibleOutputFormat, explode(
',',self::AVAILABLE_OUTPUT_FORMATS))) {
223 $this->outputFormat = $possibleOutputFormat;
226 $this->licenseMap =
new LicenseMap($this->
dbManager, $this->groupId, LicenseMap::REPORT,
true);
228 $docLicense = $this->licenseDao->getLicenseByShortName(self::DATA_LICENSE);
229 $docLicenseId = $docLicense->getId() .
"-" . md5($docLicense->getText());
231 ->setLicenseObj($docLicense)
232 ->setListedLicense(
true)
233 ->setCustomText(
false)
234 ->setTextPrinted(
true);
236 $additionalUploadIds = array_key_exists(self::UPLOAD_ADDS,
$args) ? explode(
',',
$args[self::UPLOAD_ADDS]) : array();
237 $packageIds = array($uploadId);
238 foreach ($additionalUploadIds as $additionalId) {
240 $packageIds[] = $additionalId;
242 $this->
writeReport($packageNodes, $packageIds, $uploadId);
253 $prefix = $this->outputFormat .
"-";
255 switch ($this->outputFormat) {
257 $postfix =
".xml" . $postfix;
263 $prefix .=
"copyright-";
268 $postfix =
".xml" . $postfix;
275 return $prefix . $partname . $postfix;
287 if ($this->filebasename ==
null) {
288 $fileName =
strtoupper($this->outputFormat).
"_".$packageName;
289 switch ($this->outputFormat) {
291 $fileName .=
".spdx.rdf";
294 $fileName .=
".spdx";
303 $fileName = $fileName .
".jsonld";
306 $fileName = $fileName .
".json";
309 $fileName = $fileName .
".spdx.rdf";
312 $fileName = $fileName .
".spdx";
315 $this->filebasename = $fileName;
328 $fileBase = $SysConf[
'FOSSOLOGY'][
'path'].
"/report/";
340 $url=$SysConf[
'SYSCONFIG'][
'FOSSologyURL'];
341 if (substr( $url, 0, 4 ) !==
"http") {
355 $uploadTreeTableName = $this->uploadDao->getUploadtreeTableName($uploadId);
357 $itemTreeBounds = $this->uploadDao->getParentItemBounds($uploadId,$uploadTreeTableName);
360 $filesWithLicenses = $this->reportutils
361 ->getFilesWithLicensesFromClearings($itemTreeBounds, $this->groupId,
362 $this, $this->licensesInDocument);
365 $this->reportutils->addClearingStatus($filesWithLicenses,$itemTreeBounds, $this->groupId);
368 $scannerIDs = $this->reportutils->addScannerResults($filesWithLicenses, $itemTreeBounds, $this->groupId, $this->licensesInDocument);
369 $licenseComment =
"";
370 if (!empty($scannerIDs)) {
375 $this->reportutils->addCopyrightResults($filesWithLicenses, $uploadId);
378 $upload = $this->uploadDao->getUpload($uploadId);
379 $fileNodes = $this->
generateFileNodes($filesWithLicenses, $upload->getTreeTableName(), $uploadId);
381 $mainLicenseIds = $this->clearingDao->getMainLicenseIds($uploadId, $this->groupId);
382 $mainLicenses = array();
383 foreach ($mainLicenseIds as $licId) {
384 $reportedLicenseId = $this->licenseMap->getProjectedId($licId);
385 $mainLicense = $this->licenseDao->getLicenseById($reportedLicenseId, $this->groupId);
386 if ($mainLicense ===
null) {
388 "spdx: Warning: main license ID {$reportedLicenseId} not found; skipping."
392 $reportLicId = $mainLicense->getId() .
"-" . md5($mainLicense->getText());
393 $mainLicenses[] = $reportLicId;
394 if (!array_key_exists($reportLicId, $this->licensesInDocument)) {
395 $listedLicense = stripos($mainLicense->getSpdxId(),
396 LicenseRef::SPDXREF_PREFIX) !== 0;
398 ->setLicenseObj($mainLicense)
399 ->setCustomText(
false)
400 ->setListedLicense($listedLicense);
403 $mainLicenseString = [];
404 if ($this->outputFormat ==
"spdx2tv" ||
405 $this->outputFormat ==
"spdx2csv" ||
406 $this->outputFormat ==
"spdx3tv") {
407 foreach ($mainLicenses as $mainLicense) {
408 $shortName = $this->licensesInDocument[$mainLicense]
409 ->getLicenseObj()->getShortName();
411 LicenseRef::SPDXREF_PREFIX)) {
412 $mainLicenseString[] = $shortName;
414 $mainLicenseString[] = $this->licensesInDocument[$mainLicense]
415 ->getLicenseObj()->getSpdxId();
418 if ($this->outputFormat ==
"spdx2tv") {
421 foreach ($mainLicenseString as $i => $licId) {
426 $mainLicenseString[$i] =
"LicenseRef-$licId";
434 $hashes = $this->uploadDao->getUploadHashes($uploadId);
436 $reportInfo = $this->uploadDao->getReportInfo($uploadId);
437 $componentId = $reportInfo[
'ri_component_id'];
438 $componentType = $reportInfo[
'ri_component_type'];
439 $componentVersion = $reportInfo[
'ri_version'];
440 $generalAssessment = $reportInfo[
'ri_general_assesment'];
441 $releaseDate = $reportInfo[
'ri_release_date'];
442 if ($componentId ==
"NA") {
445 if ($componentVersion ==
"NA") {
446 $componentVersion =
"";
448 if ($generalAssessment ==
"NA") {
449 $generalAssessment =
"";
451 if ($releaseDate ==
"NA") {
454 $timeStamp = strtotime($releaseDate);
455 if ($timeStamp != -1) {
456 $releaseDate = date(
"Y-m-d\\T00:00:00\\Z", $timeStamp);
462 $componentType =
"maven-central";
464 $componentType =
"purl";
466 if (!empty($componentType)) {
475 'packageId' => $uploadId,
477 'packageName' => $upload->getFilename(),
478 'packageVersion' => $componentVersion,
479 'releaseDate' => $releaseDate,
480 'generalAssessment' => $generalAssessment,
481 'uploadName' => $upload->getFilename(),
482 'componentType' => $componentType,
483 'componentId' => htmlspecialchars($componentId),
484 'sha1' => $hashes[
'sha1'],
485 'md5' => $hashes[
'md5'],
486 'sha256' => $hashes[
'sha256'],
488 'mainLicenses' => $mainLicenses,
489 'mainLicenseString' => $mainLicenseString,
490 'licenseComments' => $licenseComment,
491 'fileNodes' => $fileNodes,
492 'obligations' => $obligations,
493 'licenseList' => $this->licensesInDocument,
494 'EnableOsselotExport' => $stateOsselot
505 $func =
function($scannerId) use (
$agentDao)
507 return $agentDao->getAgentName($scannerId).
" (".
$agentDao->getAgentRev($scannerId).
")";
509 $scannerNames = array_map($func, $scannerIds);
510 return "licenseInfoInFile determined by Scanners:\n - ".implode(
"\n - ",$scannerNames);
523 if (!array_key_exists($licenses, $filesWithLicenses)) {
524 $filesWithLicenses[$licenses][
'files']=array();
525 $filesWithLicenses[$licenses][
'copyrights']=array();
527 if (empty($copyrights)) {
528 $copyrights = array();
530 $filesWithLicenses[$licenses][
'files'][$file] = $fullPath;
531 foreach ($copyrights as $copyright) {
532 if (!in_array($copyright, $filesWithLicenses[$licenses][
'copyrights'])) {
533 $filesWithLicenses[$licenses][
'copyrights'][] = $copyright;
546 $licensesWithFiles = array();
547 $treeDao = $this->container->get(
'dao.tree');
549 foreach ($filesWithLicenses as $fileId => $fileNode) {
550 $filesProceeded += 1;
551 if (($filesProceeded&2047)==0) {
554 $fullPath = $treeDao->getFullPath($fileId, $treeTableName, 0);
555 if (! empty($fileNode->getConcludedLicenses())) {
557 foreach ($fileNode->getConcludedLicenses() as $license) {
558 $licenses[] = $this->licensesInDocument[$license]
559 ->getLicenseObj()->getSpdxId();
564 $licenses, $fileNode->getCopyrights(), $fileId, $fullPath);
566 if (! empty($fileNode->getScanners())) {
567 $implodedLicenses = [];
568 foreach ($fileNode->getScanners() as $license) {
569 $implodedLicenses[] = $this->licensesInDocument[$license]
570 ->getLicenseObj()->getSpdxId();
574 if ($fileNode->isCleared()) {
575 $msgLicense =
"None (scanners found: " . $implodedLicenses .
")";
577 $msgLicense =
"NoLicenseConcluded (scanners found: " . $implodedLicenses .
")";
580 if ($fileNode->isCleared()) {
581 $msgLicense =
"None";
583 $msgLicense =
"NoLicenseConcluded";
587 $fileNode->getCopyrights(), $fileId, $fullPath);
590 return $licensesWithFiles;
599 $upload = $this->uploadDao->getUpload($uploadId);
600 $packageName = $upload->getFilename();
602 $this->uri = $this->
getUri($packageName);
603 $this->filename = $this->
getFileName($packageName);
612 protected function writeReport(&$packageNodes, $packageIds, $uploadId)
616 $fileBase = dirname($this->filename);
618 if (!is_dir($fileBase)) {
619 mkdir($fileBase, 0777,
true);
623 $organizationName = $SysConf[
'SYSCONFIG'][
"ReportHeaderText"] ??
'FOSSology';
624 $version = $SysConf[
'BUILD'][
'VERSION'];
626 $uploadTreeTableName = $this->uploadDao->getUploadtreeTableName($uploadId);
627 $itemTreeBounds = $this->uploadDao->getParentItemBounds($uploadId,$uploadTreeTableName);
630 $filesWithLicenses = $this->reportutils
631 ->getFilesWithLicensesFromClearings($itemTreeBounds, $this->groupId,
632 $this, $this->licensesInDocument);
634 $upload = $this->uploadDao->getUpload($uploadId);
635 $this->
generateFileNodes($filesWithLicenses, $upload->getTreeTableName(), $uploadId);
636 $this->declaredLicenseFileIds = array_unique(array_diff($this->declaredLicenseFileIds, $this->concludedLicenseFileIds));
637 $this->concludedLicenseFileIds = array_unique($this->concludedLicenseFileIds);
639 'documentName' => $fileBase,
641 'userName' => $this->container->get(
'dao.user')->getUserName($this->userId) .
" (" . $this->container->get(
'dao.user')->getUserEmail($this->userId) .
")",
642 'organisation' => $organizationName,
643 'concludedLicenseFileIds'=>$this->concludedLicenseFileIds,
644 'declaredLicenseFileIds'=>$this->declaredLicenseFileIds,
645 'toolVersion' =>
'fossology-' . $version,
646 'packageNodes' => $packageNodes,
647 'packageIds' => $packageIds,
648 'dataLicense' => $this->getSPDXDataLicense(),
649 'licenseList' => $this->licensesInDocument,
650 'EnableOsselotExport' => $stateOsselot,
656 $message = preg_replace(
'/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/',
'?',$message);
658 file_put_contents($this->filename, $message);
670 $this->reportutils->updateOrInsertReportgenEntry($uploadId,
$jobId, $fileName);
681 return $this->renderer->load($templateName)->render($vars);
694 if (strcmp($this->outputFormat,
"dep5") !== 0) {
711 $treeDao = $this->container->get(
'dao.tree');
715 $textToBePrinted = [];
716 foreach ($filesWithLicenses as $fileId => $fileData) {
717 $filesProceeded += 1;
718 if (($filesProceeded & 2047) == 0) {
719 $this->
heartbeat($filesProceeded - $lastValue);
720 $lastValue = $filesProceeded;
722 $hashes = $treeDao->getItemHashes($fileId);
723 $fileName = $treeDao->getFullPath($fileId, $treeTableName, 0);
727 foreach ($fileData->getConcludedLicenses() as $license) {
728 $this->concludedLicenseFileIds[] = $fileId;
729 if (! $this->licensesInDocument[$license]->isTextPrinted()) {
730 $textToBePrinted[] = $license;
733 foreach ($fileData->getScanners() as $license) {
734 $this->declaredLicenseFileIds[] = $fileId;
735 if (! $this->licensesInDocument[$license]->isTextPrinted()) {
736 $textToBePrinted[] = $license;
739 $concludedLicensesString = [];
740 if ($this->outputFormat ==
"spdx2tv" ||
741 $this->outputFormat ==
"spdx2csv" ||
742 $this->outputFormat ==
"spdx3tv") {
743 foreach ($fileData->getConcludedLicenses() as $license) {
744 $shortName = $this->licensesInDocument[$license]
745 ->getLicenseObj()->getShortName();
747 LicenseRef::SPDXREF_PREFIX)) {
748 $concludedLicensesString[] = $shortName;
750 $concludedLicensesString[] = $this->licensesInDocument[$license]
751 ->getLicenseObj()->getSpdxId();
754 if ($this->outputFormat ==
"spdx2tv") {
756 foreach ($concludedLicensesString as $j => $licId) {
761 $concludedLicensesString[$j] =
"LicenseRef-$licId";
772 if (!$stateWoInfos ||
773 ($stateWoInfos && (!empty($fileData->getConcludedLicenses()) ||
774 !empty($fileData->getScanners()) || !empty($fileData->getCopyrights())))) {
775 $fileData->setAcknowledgements(
777 $fileData->setComments(
779 $dataTemplate = array(
781 'sha1' => $hashes[
'sha1'],
782 'md5' => $hashes[
'md5'],
783 'sha256' => $hashes[
'sha256'],
785 'fileName' => $fileName,
786 'fileDirName' => dirname($fileName),
787 'fileBaseName' => basename($fileName),
788 'fileData' => $fileData,
789 'licenseList' => $this->licensesInDocument,
790 'concludedLicensesString' => $concludedLicensesString,
791 'licenseCommentState' => $stateComment,
792 'EnableOsselotExport' =>$stateOsselot,
797 foreach ($textToBePrinted as $license) {
798 $this->licensesInDocument[$license]->setTextPrinted(
true);
801 $this->
heartbeat($filesProceeded - $lastValue);
819 foreach ($licensesWithFiles as $licenseId=>$entry) {
820 $filesProceeded += count($entry[
'files']);
821 if ($filesProceeded&(~2047) > $lastStep) {
822 $this->
heartbeat($filesProceeded - $lastValue);
823 $lastStep = $filesProceeded&(~2047) + 2048;
824 $lastValue = $filesProceeded;
828 if (strrpos($licenseId,
"NoLicenseConcluded (scanners found: ", -strlen($licenseId)) !==
false) {
829 $comment = substr($licenseId,20,strlen($licenseId)-21);
830 $licenseId =
"NoLicenseConcluded";
831 } elseif (strrpos($licenseId,
"None (scanners found: ", -strlen($licenseId)) !==
false) {
832 $comment = substr($licenseId,6,strlen($licenseId)-7);
837 'fileNames'=>$entry[
'files'],
838 'license'=>$licenseId,
839 'copyrights'=>$entry[
'copyrights'],
840 'comment'=>$comment));
842 $this->
heartbeat($filesProceeded - $lastValue);
858 if ($upload->getTreeTableName()==
'uploadtree_a') {
859 $sql = $upload->getTreeTableName().
' WHERE upload_fk=$1 AND';
860 $param[] = $upload->getId();
862 $sql = $upload->getTreeTableName().
' WHERE';
863 $stmt .=
'.'.$upload->getTreeTableName();
866 $sql =
"SELECT STRING_AGG(lower_sha1,'') concat_sha1 FROM
867 (SELECT LOWER(pfile_sha1) lower_sha1 FROM pfile, $sql pfile_fk=pfile_pk AND parent IS NOT NULL ORDER BY pfile_sha1) templist";
868 $filelistPack = $this->
dbManager->getSingleRow($sql,$param,$stmt);
870 return sha1($filelistPack[
'concat_sha1']);
882 $sql =
"SELECT ri_spdx_selection FROM report_info WHERE upload_fk = $1";
883 $getCommentState = $this->
dbManager->getSingleRow($sql, array($uploadId), __METHOD__.
'.SPDX_license_comment');
884 if (!empty($getCommentState[
'ri_spdx_selection'])) {
885 $getCommentStateSingle = explode(
',', $getCommentState[
'ri_spdx_selection']);
886 if ($getCommentStateSingle[$key] ===
"checked") {
900 $licenses = $this->licenseClearedGetter->getCleared($uploadId, $this,
903 $licensesMain = $this->licenseMainGetter->getCleared($uploadId, $this,
906 list($obligations, $_) = $this->obligationsGetter->getObligations(
907 $licenses[
'statements'], $licensesMain[
'statements'], $uploadId,
909 if (empty($obligations)) {
912 return array_column($obligations,
"text");
922 $dataLic = $this->licenseDao->getLicenseByShortName(self::DATA_LICENSE);
923 return $dataLic->getId() .
"-" . md5($dataLic->getText());
939 $localList = array_values($this->licensesInDocument);
947 for ($i = 0; $i < count($localList) - 1; $i++) {
948 if ((! $localList[$i]->isCustomText() && ! $localList[$i + 1]->isCustomText()) &&
949 $localList[$i]->getLicenseObj()->getSpdxId() ===
950 $localList[$i + 1]->getLicenseObj()->getSpdxId()) {
951 $newShortName = $localList[$i + 1]->getLicenseObj()->getShortName();
953 $localList[$i + 1]->getLicenseObj()->getShortName(),
954 LicenseRef::SPDXREF_PREFIX)) {
955 $newShortName = LicenseRef::SPDXREF_PREFIX .
956 $localList[$i + 1]->getLicenseObj()->getShortName();
957 $newShortName = preg_replace(
'/\+$/',
'-or-later', $newShortName);
959 $md5 = md5($localList[$i + 1]->getLicenseObj()->getText());
960 $reportedLicenseShortname =
"$newShortName-$md5";
961 $licIndex = $localList[$i + 1]->getLicenseObj()->getId() .
"-$md5";
962 $oldLicObj = $this->licensesInDocument[$licIndex]->getLicenseObj();
963 $this->licensesInDocument[$licIndex]->setLicenseObj(
964 new License($oldLicObj->getId(), $reportedLicenseShortname,
965 $oldLicObj->getFullName(), $oldLicObj->getRisk(),
966 $oldLicObj->getText(), $oldLicObj->getUrl(),
967 $oldLicObj->getDetectorType(), $oldLicObj->getSpdxId()));
973 $agent =
new SpdxAgent();
974 $agent->scheduler_connect();
975 $agent->run_scheduler_event_loop();
976 $agent->scheduler_disconnect(0);
Structure of an Agent with all required parameters.
heartbeat($newProcessed)
Send hear beat to the scheduler.
Contains the constants and helpers for authentication of user.
Wrapper class for license map.
static stringStartsWith($haystack, $needle)
const DEFAULT_OUTPUT_FORMAT
Default output format.
getLicenseComment($scannerIds)
getFileName($packageName)
Get absolute path for report.
getVerificationCode(Upload $upload)
Get a unique identifier for a given upload.
getTemplateFile($partname)
Get TWIG template file based on output format.
const UPLOAD_ADDS
Argument for additional uploads.
getObligations(int $uploadId, int $groupId)
processUploadId($uploadId)
Given an upload ID, process the items in it.
const DATA_LICENSE
Data license for SPDX reports.
updateReportTable($uploadId, $jobId, $fileName)
Update the reportgen table with new report path.
preWorkOnArgs($args)
Parse arguments.
renderString($templateName, $vars)
Render a twig template.
const AVAILABLE_OUTPUT_FORMATS
Output formats available.
renderPackage($uploadId)
Given an upload id, render the report string.
getFileBasename($packageName)
Generate report basename based on upload name.
generateFileNodesByLicenses($filesWithLicenses, $treeTableName)
For each file, generate the nodes by licenses.
toLicensesWithFiles(&$filesWithLicenses, $treeTableName)
Map findings to the files.
generateFileNodes($filesWithLicenses, $treeTableName, $uploadId)
Generate report nodes for files.
getSPDXReportConf($uploadId, $key)
Get spdx license comment state for a given upload.
toLicensesWithFilesAdder(&$filesWithLicenses, $licenses, $copyrights, $file, $fullPath)
Map licenses, copyrights, files and full path to filesWithLicenses array.
generateFileNodesByFiles($filesWithLicenses, $treeTableName, $uploadId)
For each file, generate the nodes by files.
writeReport(&$packageNodes, $packageIds, $uploadId)
Write the report the file and update report table.
computeUri($uploadId)
For a given upload, compute the URI and filename for the report.
const OUTPUT_FORMAT_KEY
Argument key for output format.
getUri($packageName)
Get the URI for the given package.
deduplicateLicenseList()
De-duplicate license list by comparing licenses with the same SPDX ID.
static preWorkOnArgsFlp($args, $key1, $key2)
For a given set of arguments assign $args[$key1] and $args[$key2].
static implodeLicenses($licenses)
Implode licenses with "AND" or "OR".
static removeEmptyLicenses($licenses)
static cleanTextArray($texts)
char * trim(char *ptext)
Trimming whitespace.
int jobId
The id of the job.
fo_dbManager * dbManager
fo_dbManager object
FUNCTION char * strtoupper(char *s)
Helper function to upper case a string.
Namespace used by SPDX2 agent.
list_t type structure used to keep various lists. (e.g. there are multiple lists).