66 include_once(__DIR__ .
"/spdxutils.php");
68 include_once(__DIR__ .
"/version.php");
69 include_once(__DIR__ .
"/services.php");
162 function __construct()
165 $args = getopt(
"", array(self::OUTPUT_FORMAT_KEY.
'::'));
167 if (array_key_exists(self::OUTPUT_FORMAT_KEY,
$args)) {
174 parent::__construct(
$agentName, AGENT_VERSION, AGENT_REV);
176 $this->uploadDao = $this->container->get(
'dao.upload');
177 $this->clearingDao = $this->container->get(
'dao.clearing');
178 $this->licenseDao = $this->container->get(
'dao.license');
179 $this->
dbManager = $this->container->get(
'db.manager');
180 $this->renderer = $this->container->get(
'twig.environment');
181 $this->renderer->setCache(
false);
183 $this->agentSpecifLongOptions[] = self::UPLOAD_ADDS.
':';
184 $this->agentSpecifLongOptions[] = self::OUTPUT_FORMAT_KEY.
':';
199 if ((!array_key_exists(self::OUTPUT_FORMAT_KEY,
$args)
200 ||
$args[self::OUTPUT_FORMAT_KEY] ===
"")
201 && array_key_exists(self::UPLOAD_ADDS,
$args)) {
204 if (!array_key_exists(self::UPLOAD_ADDS,
$args) ||
$args[self::UPLOAD_ADDS] ===
"") {
219 if (array_key_exists(self::OUTPUT_FORMAT_KEY,
$args)) {
220 $possibleOutputFormat =
trim(
$args[self::OUTPUT_FORMAT_KEY]);
221 if (in_array($possibleOutputFormat, explode(
',',self::AVAILABLE_OUTPUT_FORMATS))) {
222 $this->outputFormat = $possibleOutputFormat;
225 $this->licenseMap =
new LicenseMap($this->
dbManager, $this->groupId, LicenseMap::REPORT,
true);
227 $docLicense = $this->licenseDao->getLicenseByShortName(self::DATA_LICENSE);
228 $docLicenseId = $docLicense->getId() .
"-" . md5($docLicense->getText());
230 ->setLicenseObj($docLicense)
231 ->setListedLicense(
true)
232 ->setCustomText(
false)
233 ->setTextPrinted(
true);
235 $additionalUploadIds = array_key_exists(self::UPLOAD_ADDS,
$args) ? explode(
',',
$args[self::UPLOAD_ADDS]) : array();
236 $packageIds = array($uploadId);
237 foreach ($additionalUploadIds as $additionalId) {
239 $packageIds[] = $additionalId;
241 $this->
writeReport($packageNodes, $packageIds, $uploadId);
252 $prefix = $this->outputFormat .
"-";
254 switch ($this->outputFormat) {
256 $postfix =
".xml" . $postfix;
262 $prefix .=
"copyright-";
267 $postfix =
".xml" . $postfix;
274 return $prefix . $partname . $postfix;
286 if ($this->filebasename ==
null) {
287 $fileName =
strtoupper($this->outputFormat).
"_".$packageName;
288 switch ($this->outputFormat) {
290 $fileName .=
".spdx.rdf";
293 $fileName .=
".spdx";
302 $fileName = $fileName .
".jsonld";
305 $fileName = $fileName .
".json";
308 $fileName = $fileName .
".spdx.rdf";
311 $fileName = $fileName .
".spdx";
314 $this->filebasename = $fileName;
327 $fileBase = $SysConf[
'FOSSOLOGY'][
'path'].
"/report/";
339 $url=$SysConf[
'SYSCONFIG'][
'FOSSologyURL'];
340 if (substr( $url, 0, 4 ) !==
"http") {
354 $uploadTreeTableName = $this->uploadDao->getUploadtreeTableName($uploadId);
355 $itemTreeBounds = $this->uploadDao->getParentItemBounds($uploadId,$uploadTreeTableName);
358 $filesWithLicenses = $this->reportutils
359 ->getFilesWithLicensesFromClearings($itemTreeBounds, $this->groupId,
360 $this, $this->licensesInDocument);
363 $this->reportutils->addClearingStatus($filesWithLicenses,$itemTreeBounds, $this->groupId);
366 $scannerIDs = $this->reportutils->addScannerResults($filesWithLicenses, $itemTreeBounds, $this->groupId, $this->licensesInDocument);
367 $licenseComment =
"";
368 if (!empty($scannerIDs)) {
373 $this->reportutils->addCopyrightResults($filesWithLicenses, $uploadId);
376 $upload = $this->uploadDao->getUpload($uploadId);
377 $fileNodes = $this->
generateFileNodes($filesWithLicenses, $upload->getTreeTableName(), $uploadId);
379 $mainLicenseIds = $this->clearingDao->getMainLicenseIds($uploadId, $this->groupId);
380 $mainLicenses = array();
381 foreach ($mainLicenseIds as $licId) {
382 $reportedLicenseId = $this->licenseMap->getProjectedId($licId);
383 $mainLicense = $this->licenseDao->getLicenseById($reportedLicenseId, $this->groupId);
384 if ($mainLicense ===
null) {
386 "spdx: Error: main license ID {$reportedLicenseId} not found; skipping."
390 $reportLicId = $mainLicense->getId() .
"-" . md5($mainLicense->getText());
391 $mainLicenses[] = $reportLicId;
392 if (!array_key_exists($reportLicId, $this->licensesInDocument)) {
393 $listedLicense = stripos($mainLicense->getSpdxId(),
394 LicenseRef::SPDXREF_PREFIX) !== 0;
396 ->setLicenseObj($mainLicense)
397 ->setCustomText(
false)
398 ->setListedLicense($listedLicense);
401 $mainLicenseString = [];
402 if ($this->outputFormat ==
"spdx2tv" ||
403 $this->outputFormat ==
"spdx2csv" ||
404 $this->outputFormat ==
"spdx3tv") {
405 foreach ($mainLicenses as $mainLicense) {
406 $shortName = $this->licensesInDocument[$mainLicense]
407 ->getLicenseObj()->getShortName();
409 LicenseRef::SPDXREF_PREFIX)) {
410 $mainLicenseString[] = $shortName;
412 $mainLicenseString[] = $this->licensesInDocument[$mainLicense]
413 ->getLicenseObj()->getSpdxId();
420 $hashes = $this->uploadDao->getUploadHashes($uploadId);
422 $reportInfo = $this->uploadDao->getReportInfo($uploadId);
423 $componentId = $reportInfo[
'ri_component_id'];
424 $componentType = $reportInfo[
'ri_component_type'];
425 $componentVersion = $reportInfo[
'ri_version'];
426 $generalAssessment = $reportInfo[
'ri_general_assesment'];
427 $releaseDate = $reportInfo[
'ri_release_date'];
428 if ($componentId ==
"NA") {
431 if ($componentVersion ==
"NA") {
432 $componentVersion =
"";
434 if ($generalAssessment ==
"NA") {
435 $generalAssessment =
"";
437 if ($releaseDate ==
"NA") {
440 $timeStamp = strtotime($releaseDate);
441 if ($timeStamp != -1) {
442 $releaseDate = date(
"Y-m-d\\T00:00:00\\Z", $timeStamp);
448 $componentType =
"maven-central";
450 $componentType =
"purl";
452 if (!empty($componentType)) {
461 'packageId' => $uploadId,
463 'packageName' => $upload->getFilename(),
464 'packageVersion' => $componentVersion,
465 'releaseDate' => $releaseDate,
466 'generalAssessment' => $generalAssessment,
467 'uploadName' => $upload->getFilename(),
468 'componentType' => $componentType,
469 'componentId' => htmlspecialchars($componentId),
470 'sha1' => $hashes[
'sha1'],
471 'md5' => $hashes[
'md5'],
472 'sha256' => $hashes[
'sha256'],
474 'mainLicenses' => $mainLicenses,
475 'mainLicenseString' => $mainLicenseString,
476 'licenseComments' => $licenseComment,
477 'fileNodes' => $fileNodes,
478 'obligations' => $obligations,
479 'licenseList' => $this->licensesInDocument
490 $func =
function($scannerId) use (
$agentDao)
492 return $agentDao->getAgentName($scannerId).
" (".
$agentDao->getAgentRev($scannerId).
")";
494 $scannerNames = array_map($func, $scannerIds);
495 return "licenseInfoInFile determined by Scanners:\n - ".implode(
"\n - ",$scannerNames);
508 if (!array_key_exists($licenses, $filesWithLicenses)) {
509 $filesWithLicenses[$licenses][
'files']=array();
510 $filesWithLicenses[$licenses][
'copyrights']=array();
512 if (empty($copyrights)) {
513 $copyrights = array();
515 $filesWithLicenses[$licenses][
'files'][$file] = $fullPath;
516 foreach ($copyrights as $copyright) {
517 if (!in_array($copyright, $filesWithLicenses[$licenses][
'copyrights'])) {
518 $filesWithLicenses[$licenses][
'copyrights'][] = $copyright;
531 $licensesWithFiles = array();
532 $treeDao = $this->container->get(
'dao.tree');
534 foreach ($filesWithLicenses as $fileId => $fileNode) {
535 $filesProceeded += 1;
536 if (($filesProceeded&2047)==0) {
539 $fullPath = $treeDao->getFullPath($fileId, $treeTableName, 0);
540 if (! empty($fileNode->getConcludedLicenses())) {
542 foreach ($fileNode->getConcludedLicenses() as $license) {
543 $licenses[] = $this->licensesInDocument[$license]
544 ->getLicenseObj()->getSpdxId();
549 $licenses, $fileNode->getCopyrights(), $fileId, $fullPath);
551 if (! empty($fileNode->getScanners())) {
552 $implodedLicenses = [];
553 foreach ($fileNode->getScanners() as $license) {
554 $implodedLicenses[] = $this->licensesInDocument[$license]
555 ->getLicenseObj()->getSpdxId();
559 if ($fileNode->isCleared()) {
560 $msgLicense =
"None (scanners found: " . $implodedLicenses .
")";
562 $msgLicense =
"NoLicenseConcluded (scanners found: " . $implodedLicenses .
")";
565 if ($fileNode->isCleared()) {
566 $msgLicense =
"None";
568 $msgLicense =
"NoLicenseConcluded";
572 $fileNode->getCopyrights(), $fileId, $fullPath);
575 return $licensesWithFiles;
584 $upload = $this->uploadDao->getUpload($uploadId);
585 $packageName = $upload->getFilename();
587 $this->uri = $this->
getUri($packageName);
588 $this->filename = $this->
getFileName($packageName);
597 protected function writeReport(&$packageNodes, $packageIds, $uploadId)
601 $fileBase = dirname($this->filename);
603 if (!is_dir($fileBase)) {
604 mkdir($fileBase, 0777,
true);
608 $organizationName = $SysConf[
'SYSCONFIG'][
"ReportHeaderText"] ??
'FOSSology';
609 $version = $SysConf[
'BUILD'][
'VERSION'];
611 $uploadTreeTableName = $this->uploadDao->getUploadtreeTableName($uploadId);
612 $itemTreeBounds = $this->uploadDao->getParentItemBounds($uploadId,$uploadTreeTableName);
615 $filesWithLicenses = $this->reportutils
616 ->getFilesWithLicensesFromClearings($itemTreeBounds, $this->groupId,
617 $this, $this->licensesInDocument);
619 $upload = $this->uploadDao->getUpload($uploadId);
620 $this->
generateFileNodes($filesWithLicenses, $upload->getTreeTableName(), $uploadId);
621 $this->declaredLicenseFileIds = array_unique(array_diff($this->declaredLicenseFileIds, $this->concludedLicenseFileIds));
622 $this->concludedLicenseFileIds = array_unique($this->concludedLicenseFileIds);
624 'documentName' => $fileBase,
626 'userName' => $this->container->get(
'dao.user')->getUserName($this->userId) .
" (" . $this->container->get(
'dao.user')->getUserEmail($this->userId) .
")",
627 'organisation' => $organizationName,
628 'concludedLicenseFileIds'=>$this->concludedLicenseFileIds,
629 'declaredLicenseFileIds'=>$this->declaredLicenseFileIds,
630 'toolVersion' =>
'fossology-' . $version,
631 'packageNodes' => $packageNodes,
632 'packageIds' => $packageIds,
633 'dataLicense' => $this->getSPDXDataLicense(),
634 'licenseList' => $this->licensesInDocument
640 $message = preg_replace(
'/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/',
'?',$message);
642 file_put_contents($this->filename, $message);
654 $this->reportutils->updateOrInsertReportgenEntry($uploadId,
$jobId, $fileName);
665 return $this->renderer->load($templateName)->render($vars);
678 if (strcmp($this->outputFormat,
"dep5") !== 0) {
695 $treeDao = $this->container->get(
'dao.tree');
699 $textToBePrinted = [];
700 foreach ($filesWithLicenses as $fileId => $fileData) {
701 $filesProceeded += 1;
702 if (($filesProceeded & 2047) == 0) {
703 $this->
heartbeat($filesProceeded - $lastValue);
704 $lastValue = $filesProceeded;
706 $hashes = $treeDao->getItemHashes($fileId);
707 $fileName = $treeDao->getFullPath($fileId, $treeTableName, 0);
710 foreach ($fileData->getConcludedLicenses() as $license) {
711 $this->concludedLicenseFileIds[] = $fileId;
712 if (! $this->licensesInDocument[$license]->isTextPrinted()) {
713 $textToBePrinted[] = $license;
716 foreach ($fileData->getScanners() as $license) {
717 $this->declaredLicenseFileIds[] = $fileId;
718 if (! $this->licensesInDocument[$license]->isTextPrinted()) {
719 $textToBePrinted[] = $license;
722 $concludedLicensesString = [];
723 if ($this->outputFormat ==
"spdx2tv" ||
724 $this->outputFormat ==
"spdx2csv" ||
725 $this->outputFormat ==
"spdx3tv") {
726 foreach ($fileData->getConcludedLicenses() as $license) {
727 $shortName = $this->licensesInDocument[$license]
728 ->getLicenseObj()->getShortName();
730 LicenseRef::SPDXREF_PREFIX)) {
731 $concludedLicensesString[] = $shortName;
733 $concludedLicensesString[] = $this->licensesInDocument[$license]
734 ->getLicenseObj()->getSpdxId();
740 if (!$stateWoInfos ||
741 ($stateWoInfos && (!empty($fileData->getConcludedLicenses()) ||
742 !empty($fileData->getScanners()) || !empty($fileData->getCopyrights())))) {
743 $fileData->setAcknowledgements(
745 $fileData->setComments(
747 $dataTemplate = array(
749 'sha1' => $hashes[
'sha1'],
750 'md5' => $hashes[
'md5'],
751 'sha256' => $hashes[
'sha256'],
753 'fileName' => $fileName,
754 'fileDirName' => dirname($fileName),
755 'fileBaseName' => basename($fileName),
756 'fileData' => $fileData,
757 'licenseList' => $this->licensesInDocument,
758 'concludedLicensesString' => $concludedLicensesString,
759 'licenseCommentState' => $stateComment
764 foreach ($textToBePrinted as $license) {
765 $this->licensesInDocument[$license]->setTextPrinted(
true);
768 $this->
heartbeat($filesProceeded - $lastValue);
786 foreach ($licensesWithFiles as $licenseId=>$entry) {
787 $filesProceeded += count($entry[
'files']);
788 if ($filesProceeded&(~2047) > $lastStep) {
789 $this->
heartbeat($filesProceeded - $lastValue);
790 $lastStep = $filesProceeded&(~2047) + 2048;
791 $lastValue = $filesProceeded;
795 if (strrpos($licenseId,
"NoLicenseConcluded (scanners found: ", -strlen($licenseId)) !==
false) {
796 $comment = substr($licenseId,20,strlen($licenseId)-21);
797 $licenseId =
"NoLicenseConcluded";
798 } elseif (strrpos($licenseId,
"None (scanners found: ", -strlen($licenseId)) !==
false) {
799 $comment = substr($licenseId,6,strlen($licenseId)-7);
804 'fileNames'=>$entry[
'files'],
805 'license'=>$licenseId,
806 'copyrights'=>$entry[
'copyrights'],
807 'comment'=>$comment));
809 $this->
heartbeat($filesProceeded - $lastValue);
825 if ($upload->getTreeTableName()==
'uploadtree_a') {
826 $sql = $upload->getTreeTableName().
' WHERE upload_fk=$1 AND';
827 $param[] = $upload->getId();
829 $sql = $upload->getTreeTableName().
' WHERE';
830 $stmt .=
'.'.$upload->getTreeTableName();
833 $sql =
"SELECT STRING_AGG(lower_sha1,'') concat_sha1 FROM
834 (SELECT LOWER(pfile_sha1) lower_sha1 FROM pfile, $sql pfile_fk=pfile_pk AND parent IS NOT NULL ORDER BY pfile_sha1) templist";
835 $filelistPack = $this->
dbManager->getSingleRow($sql,$param,$stmt);
837 return sha1($filelistPack[
'concat_sha1']);
848 $sql =
"SELECT ri_spdx_selection FROM report_info WHERE upload_fk = $1";
849 $getCommentState = $this->
dbManager->getSingleRow($sql, array($uploadId), __METHOD__.
'.SPDX_license_comment');
850 if (!empty($getCommentState[
'ri_spdx_selection'])) {
851 $getCommentStateSingle = explode(
',', $getCommentState[
'ri_spdx_selection']);
852 if ($getCommentStateSingle[$key] ===
"checked") {
867 $licenses = $this->licenseClearedGetter->getCleared($uploadId, $this,
870 $licensesMain = $this->licenseMainGetter->getCleared($uploadId, $this,
873 list($obligations, $_) = $this->obligationsGetter->getObligations(
874 $licenses[
'statements'], $licensesMain[
'statements'], $uploadId,
876 if (empty($obligations)) {
879 return array_column($obligations,
"text");
889 $dataLic = $this->licenseDao->getLicenseByShortName(self::DATA_LICENSE);
890 return $dataLic->getId() .
"-" . md5($dataLic->getText());
906 $localList = array_values($this->licensesInDocument);
914 for ($i = 0; $i < count($localList) - 1; $i++) {
915 if ((! $localList[$i]->isCustomText() && ! $localList[$i + 1]->isCustomText()) &&
916 $localList[$i]->getLicenseObj()->getSpdxId() ===
917 $localList[$i + 1]->getLicenseObj()->getSpdxId()) {
918 $newShortName = $localList[$i + 1]->getLicenseObj()->getShortName();
920 $localList[$i + 1]->getLicenseObj()->getShortName(),
921 LicenseRef::SPDXREF_PREFIX)) {
922 $newShortName = LicenseRef::SPDXREF_PREFIX .
923 $localList[$i + 1]->getLicenseObj()->getShortName();
924 $newShortName = preg_replace(
'/\+$/',
'-or-later', $newShortName);
926 $md5 = md5($localList[$i + 1]->getLicenseObj()->getText());
927 $reportedLicenseShortname =
"$newShortName-$md5";
928 $licIndex = $localList[$i + 1]->getLicenseObj()->getId() .
"-$md5";
929 $oldLicObj = $this->licensesInDocument[$licIndex]->getLicenseObj();
930 $this->licensesInDocument[$licIndex]->setLicenseObj(
931 new License($oldLicObj->getId(), $reportedLicenseShortname,
932 $oldLicObj->getFullName(), $oldLicObj->getRisk(),
933 $oldLicObj->getText(), $oldLicObj->getUrl(),
934 $oldLicObj->getDetectorType(), $oldLicObj->getSpdxId()));
940 $agent =
new SpdxAgent();
941 $agent->scheduler_connect();
942 $agent->run_scheduler_event_loop();
943 $agent->scheduler_disconnect(0);
Structure of an Agent with all required parameters.
heartbeat($newProcessed)
Send hear beat to the scheduler.
Wrapper class for license map.
static stringStartsWith($haystack, $needle)
const DEFAULT_OUTPUT_FORMAT
Default output format.
getLicenseComment($scannerIds)
getFileName($packageName)
Get absolute path for report.
getVerificationCode(Upload $upload)
Get a unique identifier for a given upload.
getTemplateFile($partname)
Get TWIG template file based on output format.
const UPLOAD_ADDS
Argument for additional uploads.
getObligations(int $uploadId, int $groupId)
processUploadId($uploadId)
Given an upload ID, process the items in it.
const DATA_LICENSE
Data license for SPDX reports.
updateReportTable($uploadId, $jobId, $fileName)
Update the reportgen table with new report path.
preWorkOnArgs($args)
Parse arguments.
renderString($templateName, $vars)
Render a twig template.
const AVAILABLE_OUTPUT_FORMATS
Output formats available.
renderPackage($uploadId)
Given an upload id, render the report string.
getFileBasename($packageName)
Generate report basename based on upload name.
generateFileNodesByLicenses($filesWithLicenses, $treeTableName)
For each file, generate the nodes by licenses.
toLicensesWithFiles(&$filesWithLicenses, $treeTableName)
Map findings to the files.
generateFileNodes($filesWithLicenses, $treeTableName, $uploadId)
Generate report nodes for files.
getSPDXReportConf($uploadId, $key)
Get spdx license comment state for a given upload.
toLicensesWithFilesAdder(&$filesWithLicenses, $licenses, $copyrights, $file, $fullPath)
Map licenses, copyrights, files and full path to filesWithLicenses array.
generateFileNodesByFiles($filesWithLicenses, $treeTableName, $uploadId)
For each file, generate the nodes by files.
writeReport(&$packageNodes, $packageIds, $uploadId)
Write the report the file and update report table.
computeUri($uploadId)
For a given upload, compute the URI and filename for the report.
const OUTPUT_FORMAT_KEY
Argument key for output format.
getUri($packageName)
Get the URI for the given package.
deduplicateLicenseList()
De-duplicate license list by comparing licenses with the same SPDX ID.
static preWorkOnArgsFlp($args, $key1, $key2)
For a given set of arguments assign $args[$key1] and $args[$key2].
static implodeLicenses($licenses)
Implode licenses with "AND" or "OR".
static removeEmptyLicenses($licenses)
static cleanTextArray($texts)
char * trim(char *ptext)
Trimming whitespace.
int jobId
The id of the job.
fo_dbManager * dbManager
fo_dbManager object
FUNCTION char * strtoupper(char *s)
Helper function to upper case a string.
Namespace used by SPDX2 agent.
list_t type structure used to keep various lists. (e.g. there are multiple lists).