61 use Symfony\Component\Process\Process;
63 include_once(__DIR__ .
"/version.php");
71 const RULES_NOMOS_IN_MONK = 0x1;
72 const RULES_NOMOS_MONK_NINKA = 0x2;
73 const RULES_BULK_REUSE = 0x4;
74 const RULES_WIP_SCANNER_UPDATES = 0x8;
75 const RULES_OJO_NO_CONTRADICTION = 0x10;
76 const RULES_COPYRIGHT_FALSE_POSITIVE = 0x20;
77 const RULES_COPYRIGHT_FALSE_POSITIVE_CLUTTER = 0x40;
78 const RULES_ALL = 0xff;
128 parent::__construct(AGENT_DECIDER_NAME, AGENT_DECIDER_VERSION, AGENT_DECIDER_REV);
130 $this->uploadDao = $this->container->get(
'dao.upload');
131 $this->clearingDao = $this->container->get(
'dao.clearing');
132 $this->highlightDao = $this->container->get(
'dao.highlight');
133 $this->showJobsDao = $this->container->get(
'dao.show_jobs');
134 $this->decisionTypes = $this->container->get(
'decision.types');
135 $this->clearingDecisionProcessor = $this->container->get(
'businessrules.clearing_decision_processor');
136 $this->agentLicenseEventProcessor = $this->container->get(
'businessrules.agent_license_event_processor');
137 $this->copyrightDao = $this->container->get(
'dao.copyright');
139 $this->agentSpecifOptions =
"r:";
149 $this->activeRules = array_key_exists(
'r',
$args) ? intval(
$args[
'r']) : self::RULES_ALL;
150 $this->licenseMap =
new LicenseMap($this->
dbManager, $this->groupId, $this->licenseMapUsage);
152 if (array_key_exists(
"r",
$args) && (($this->activeRules&self::RULES_COPYRIGHT_FALSE_POSITIVE)== self::RULES_COPYRIGHT_FALSE_POSITIVE)) {
155 if (array_key_exists(
"r",
$args) && (($this->activeRules&self::RULES_COPYRIGHT_FALSE_POSITIVE_CLUTTER)== self::RULES_COPYRIGHT_FALSE_POSITIVE_CLUTTER)) {
158 if (array_key_exists(
"r",
$args) && (($this->activeRules&self::RULES_BULK_REUSE)== self::RULES_BULK_REUSE)) {
160 $bulkIds = $this->clearingDao->getPreviousBulkIds($uploadId, $this->groupId, $this->userId);
161 if (count($bulkIds) == 0) {
167 foreach ($bulkIds as $bulkId) {
168 $jqId = $bulkReuser->rerunBulkAndDeciderOnUpload($uploadId, $this->groupId, $this->userId, $bulkId, $jqId);
171 $jqIdRow = $this->showJobsDao->getDataForASingleJob($jqId);
172 while ($this->showJobsDao->getJobStatus($jqId)) {
174 $timeInSec = $this->showJobsDao->getEstimatedTime($jqIdRow[
'jq_job_fk'],
'',0,0,1);
175 if ($timeInSec > $maxTime) {
177 }
else if ($timeInSec < $minTime) {
186 $parentBounds = $this->uploadDao->getParentItemBounds($uploadId);
187 foreach ($this->uploadDao->getContainedItems($parentBounds) as $item) {
205 $itemTreeBounds = $item->getItemTreeBounds();
207 $unMappedMatches = $this->agentLicenseEventProcessor->getLatestScannerDetectedMatches($itemTreeBounds);
210 $lastDecision = $this->clearingDao->getRelevantClearingDecision($itemTreeBounds, $this->groupId);
212 if (
null!==$lastDecision && $lastDecision->getType()==DecisionTypes::IRRELEVANT) {
216 $currentEvents = $this->clearingDao->getRelevantClearingEvents($itemTreeBounds, $this->groupId);
219 if (
null !== $lastDecision && $projectedScannerMatches
220 && ($this->activeRules & self::RULES_WIP_SCANNER_UPDATES) == self::RULES_WIP_SCANNER_UPDATES) {
221 $licensesFromDecision = array();
222 foreach ($lastDecision->getClearingLicenses() as $clearingLicense) {
223 $licenseIdFromEvent = $this->licenseMap->getProjectedId($clearingLicense->getLicenseId());
224 $licensesFromDecision[$licenseIdFromEvent] = $licenseIdFromEvent;
229 if (
null !== $lastDecision && $markAsWip) {
230 $this->clearingDao->markDecisionAsWip($item->getId(), $this->userId, $this->groupId);
234 if (
null!==$lastDecision || 0<count($currentEvents)) {
238 $haveDecided =
false;
240 if (($this->activeRules&self::RULES_OJO_NO_CONTRADICTION) == self::RULES_OJO_NO_CONTRADICTION) {
244 if (!$haveDecided && ($this->activeRules&self::RULES_OJO_NO_CONTRADICTION) == self::RULES_OJO_NO_CONTRADICTION) {
248 if (!$haveDecided && ($this->activeRules&self::RULES_NOMOS_IN_MONK) == self::RULES_NOMOS_IN_MONK) {
252 if (!$haveDecided && ($this->activeRules&self::RULES_NOMOS_MONK_NINKA)== self::RULES_NOMOS_MONK_NINKA) {
256 if (!$haveDecided && $markAsWip) {
257 $this->clearingDao->markDecisionAsWip($item->getId(), $this->userId, $this->groupId);
260 return ($haveDecided||$markAsWip ? 1 : 0);
271 foreach (array_keys($projectedScannerMatches) as $projectedLicenseId) {
272 if (!array_key_exists($projectedLicenseId, $licensesFromDecision)) {
289 $licenseMatchExists = count($matches) > 0;
290 foreach ($matches as $licenseMatches) {
294 if ($licenseMatchExists) {
296 $this->clearingDecisionProcessor->makeDecisionFromLastEvents(
297 $itemTreeBounds, $this->userId, $this->groupId,
298 DecisionTypes::IDENTIFIED,
false);
299 }
catch (\Exception $e) {
300 echo
"Can not auto decide as file '" .
301 $itemTreeBounds->
getItemId() .
"' contains candidate license.\n";
304 return $licenseMatchExists;
317 $licenseMatchExists = count($matches) > 0;
318 foreach ($matches as $licenseMatches) {
322 if ($licenseMatchExists) {
324 $this->clearingDecisionProcessor->makeDecisionFromLastEvents(
325 $itemTreeBounds, $this->userId, $this->groupId,
326 DecisionTypes::IDENTIFIED,
false);
327 }
catch (\Exception $e) {
328 echo
"Can not auto decide as file '" .
329 $itemTreeBounds->
getItemId() .
"' contains candidate license.\n";
332 return $licenseMatchExists;
345 $canDecide = (count($matches)>0);
347 foreach ($matches as $licenseMatches) {
355 $this->clearingDecisionProcessor->makeDecisionFromLastEvents($itemTreeBounds, $this->userId, $this->groupId, DecisionTypes::IDENTIFIED, $global=
true);
370 $canDecide = (count($matches)>0);
372 foreach ($matches as $licenseMatches) {
380 $this->clearingDecisionProcessor->makeDecisionFromLastEvents($itemTreeBounds, $this->userId, $this->groupId, DecisionTypes::IDENTIFIED, $global=
true);
394 foreach ($matches as $licenseId => $licenseMatches) {
395 $projectedId = $this->licenseMap->getProjectedId($licenseId);
397 foreach ($licenseMatches as $agent => $agentMatches) {
398 $haveId = array_key_exists($projectedId, $remapped);
399 $haveAgent = $haveId && array_key_exists($agent, $remapped[$projectedId]);
401 $remapped[$projectedId][$agent] = array_merge($remapped[$projectedId][$agent], $agentMatches);
403 $remapped[$projectedId][$agent] = $agentMatches;
418 return ($big[0] >= 0) && ($small[0] >= $big[0]) && ($small[1] <= $big[1]);
428 if (!array_key_exists(
"nomos", $licenseMatches)) {
431 if (!array_key_exists(
"monk", $licenseMatches)) {
435 foreach ($licenseMatches[
"nomos"] as $licenseMatch) {
436 $matchId = $licenseMatch->getLicenseFileId();
437 $nomosRegion = $this->highlightDao->getHighlightRegion($matchId);
440 foreach ($licenseMatches[
"monk"] as $monkLicenseMatch) {
441 $monkRegion = $this->highlightDao->getHighlightRegion($monkLicenseMatch->getLicenseFileId());
462 $scanners = array(
'nomos',
'monk',
'ninka');
464 foreach ($scanners as $scanner) {
465 if (!array_key_exists($scanner, $licenseMatches)) {
468 foreach ($licenseMatches[$scanner] as $licenseMatch) {
469 $licId = $licenseMatch->getLicenseId();
470 $vote[$licId][$scanner] =
true;
474 foreach ($vote as $licId=>$voters) {
475 if (count($voters) != 3) {
490 if (array_key_exists($scanner, $licenseMatches) ===
true) {
493 return $match->getLicenseId();
494 }, $licenseMatches[$scanner]);
507 if (count($findingsByOjo) == 0) {
513 if (count($findingsByOtherScanner) == 0) {
517 foreach ($findingsByOtherScanner as $findingsByScanner) {
518 if (in_array($findingsByScanner, $findingsByOjo) ===
false) {
534 if (count($findingsByReso) == 0) {
540 if (count($findingsByOtherScanner) == 0) {
544 foreach ($findingsByOtherScanner as $findingsByScanner) {
545 if (in_array($findingsByScanner, $findingsByReso) ===
false) {
562 if (empty($uploadId)) {
566 $uploadTreeTableName = $this->uploadDao->getUploadtreeTableName($uploadId);
568 $scanJobProxy->createAgentStatus(array(
$agentName));
569 $selectedScanners = $scanJobProxy->getLatestSuccessfulAgentIds();
570 if (!array_key_exists(
$agentName, $selectedScanners)) {
573 $latestXpAgentId = $selectedScanners[
$agentName];
574 $extrawhere =
' agent_fk='.$latestXpAgentId;
575 $allCopyrights = $this->copyrightDao->getScannerEntries(
'copyright',
576 $uploadTreeTableName, $uploadId,
null, $extrawhere);
578 $copyrightJSON = json_encode($allCopyrights);
579 $tmpFile = tmpfile();
580 $tmpFilePath = stream_get_meta_data($tmpFile)[
'uri'];
581 fwrite($tmpFile, $copyrightJSON);
583 if (empty($deactivatedCopyrightData)) {
587 $deactivatedCopyrights = json_decode($deactivatedCopyrightData,
true);
588 foreach ($deactivatedCopyrights as $deactivatedCopyright) {
589 $item = $deactivatedCopyright[
'uploadtree_pk'];
590 $itemTreeBounds = $this->uploadDao->getItemTreeBounds($item, $uploadTreeTableName);
591 $hash = $deactivatedCopyright[
'hash'];
592 $content = $deactivatedCopyright[
'content'];
593 $cpTable =
'copyright';
594 if ($deactivatedCopyright[
'is_copyright'] ==
"t") {
596 if (array_key_exists(
'decluttered_content', $deactivatedCopyright) &&
597 !empty($deactivatedCopyright[
'decluttered_content'])) {
598 $content = $deactivatedCopyright[
'decluttered_content'];
607 $this->copyrightDao->updateTable($itemTreeBounds, $hash, $content, $this->userId, $cpTable, $action);
620 $clutter_flag): string
622 $script =
"copyrightDeactivationClutterRemovalScript.py";
623 $args = [
"python3", __DIR__ .
"/$script",
"--file", $tmpFilePath];
625 $args[] =
"--clutter";
631 $process =
new Process(
$args);
632 $process->setTimeout(
null);
639 if ($sleepTime < $maxSleepTime) {
642 }
while ($process->isRunning());
644 echo $process->getErrorOutput();
646 return $process->getOutput();
Prepares bulk licenses for an upload and run DeciderJob on it.
Agent to decide license findings in an upload.
$agentLicenseEventProcessor
existsUnhandledMatch($projectedScannerMatches, $licensesFromDecision)
Check if matches contains unhandled match.
areOtherScannerFindingsAndRESOAgreed($licenseMatches)
Check if the finding by only contains one single license and that no other scanner (nomos) has produc...
$clearingDecisionProcessor
autodecideIfResoMatchesNoContradiction(ItemTreeBounds $itemTreeBounds, $matches)
Auto decide matches which are in nomos, monk, OJO and Reso findings.
processItem(Item $item)
Given an item, check with the $activeRules and apply rules to it.
areOtherScannerFindingsAndOJOAgreed($licenseMatches)
Check if the finding by only contains one single license and that no other scanner (nomos) has produc...
callCopyrightDeactivationClutterRemovalScript($tmpFilePath, $clutter_flag)
autodecideNomosMatchesInsideMonk(ItemTreeBounds $itemTreeBounds, $matches)
Auto decide matches by nomos which are in monk findings.
autodecideNomosMonkNinka(ItemTreeBounds $itemTreeBounds, $matches)
Auto decide matches which are in nomos, monk and ninka findings.
processUploadId($uploadId)
Given an upload ID, process the items in it.
getLicenseIdsOfMatchesForScanner($scanner, $licenseMatches)
extracts the matches corresponding to a scanner from a $licenseMatches structure
isRegionIncluded($small, $big)
Check if the small highlight region is inside big one.
autodecideIfOjoMatchesNoContradiction(ItemTreeBounds $itemTreeBounds, $matches)
Auto decide matches which are in nomos, monk and OJO findings.
areNomosMatchesInsideAMonkMatch($licenseMatches)
Check if matches by nomos are inside monk findings.
getCopyrightsToDisableFalsePositivesClutter($uploadId, $clutter_flag)
remapByProjectedId($matches)
Given a set of matches, remap according to project id instead of license id.
areNomosMonkNinkaAgreed($licenseMatches)
Check if findings by all agents are same or not.
Structure of an Agent with all required parameters.
heartbeat($newProcessed)
Send hear beat to the scheduler.
Handle events related to license findings.
Utility functions to process ClearingDecision.
Wrapper class for license map.
fo_dbManager * dbManager
fo_dbManager object
Namespace for decider agent.
$GLOBALS['xyyzzzDeciderJob']