7 #include "ReuserDatabaseHandler.hpp"
16 #include <sys/socket.h>
20 #include <unicode/unistr.h>
23 #include "libfossagent.h"
45 constexpr
int DT_WIP = 0;
46 constexpr
int DT_TO_BE_DISCUSSED = 3;
47 constexpr
int DT_IRRELEVANT = 4;
48 constexpr
int DT_IDENTIFIED = 5;
49 constexpr
int DT_DO_NOT_USE = 6;
50 constexpr
int DT_NON_FUNCTIONAL = 7;
57 case DT_IDENTIFIED:
return 5;
58 case DT_DO_NOT_USE:
return 4;
59 case DT_NON_FUNCTIONAL:
return 3;
60 case DT_IRRELEVANT:
return 2;
61 case DT_TO_BE_DISCUSSED:
return 1;
68 if (
s.empty())
return false;
70 if (!((c >=
'a' && c <=
'z') || (c >=
'A' && c <=
'Z') ||
71 (c >=
'0' && c <=
'9') || c ==
'_'))
77 const std::string& input)
79 icu::UnicodeString us = icu::UnicodeString::fromUTF8(input);
80 icu::UnicodeString result;
81 for (int32_t i = 0; i < us.length(); ++i)
83 UChar32 cp = us.char32At(i);
85 bool isControl = (cp <= 0x08)
88 || (cp >= 0x0E && cp <= 0x1F)
89 || (cp >= 0x7F && cp <= 0x9F);
94 result.toUTF8String(out);
98 std::string ReuserDatabaseHandler::shellEscape(
const std::string&
s)
102 r += (c ==
'\'') ? std::string(
"'\\''") : std::string(1, c);
107 int ReuserDatabaseHandler::diffLineCount(
const std::string& a,
108 const std::string& b)
110 if (a.empty() || b.empty())
return -1;
114 std::string cmd =
"diff -- " + shellEscape(a) +
" " + shellEscape(b)
116 FILE* pipe = popen(cmd.c_str(),
"r");
117 if (!pipe)
return -1;
121 while (fgets(buf,
sizeof(buf), pipe))
124 int status = pclose(pipe);
126 if (WIFEXITED(status) && WEXITSTATUS(status) == 2)
136 if (!pfileName)
return {};
139 if (!filePath)
return {};
140 std::string result(filePath);
153 bool needsUploadFilter =
154 (table ==
"uploadtree" || table ==
"uploadtree_a");
159 "SELECT uploadtree_pk, upload_fk, lft, rgt"
160 " FROM %s WHERE parent IS NULL AND upload_fk=%d",
161 table.c_str(), uploadId)
163 "SELECT uploadtree_pk, upload_fk, lft, rgt"
164 " FROM %s WHERE parent IS NULL",
167 if (!result || result.getRowCount() == 0)
return false;
169 auto row = result.getRow(0);
170 out.uploadtree_pk = std::stoi(row[0]);
171 out.uploadTreeTableName = table;
172 out.upload_fk = std::stoi(row[1]);
173 out.lft = std::stoi(row[2]);
174 out.rgt = std::stoi(row[3]);
181 int uploadId,
int groupId)
183 std::vector<ReuseTriple> result;
187 "reuserGetReusedUploads",
188 "SELECT reused_upload_fk, reused_group_fk, reuse_mode"
190 " WHERE upload_fk=$1 AND group_fk=$2"
191 " ORDER BY date_added DESC",
198 result.push_back({std::stoi(row[0]), std::stoi(row[1]),
205 int uploadId,
int groupId)
207 std::map<int, int> result;
212 bool needsUploadFilter =
213 (table ==
"uploadtree" || table ==
"uploadtree_a");
216 bool applyGlobal =
true;
219 "reuserGetGlobalDecision",
222 "SELECT (ri_globaldecision != 0) FROM report_info WHERE upload_fk=$1",
229 std::string joinCond =
231 ?
"(ut.pfile_fk = cd.pfile_fk AND cd.scope = 1)"
232 " OR (ut.uploadtree_pk = cd.uploadtree_fk"
233 " AND cd.scope = 0 AND cd.group_fk = " + std::to_string(groupId) +
")"
234 :
"(ut.uploadtree_pk = cd.uploadtree_fk"
235 " AND cd.group_fk = " + std::to_string(groupId) +
")";
237 std::string uploadFilter =
239 ?
" AND ut.upload_fk = " + std::to_string(uploadId)
246 " SELECT DISTINCT ON(ut.uploadtree_pk)"
247 " cd.clearing_decision_pk AS id,"
248 " cd.pfile_fk AS pfile_id,"
249 " cd.decision_type AS dec_type"
250 " FROM clearing_decision cd"
251 " INNER JOIN %s ut ON (%s)%s"
252 " WHERE cd.decision_type != 0"
253 " ORDER BY ut.uploadtree_pk, cd.scope ASC,"
254 " cd.clearing_decision_pk DESC"
257 " SELECT id, pfile_id, dec_type"
259 " ORDER BY pfile_id, id DESC"
261 " SELECT id, pfile_id, dec_type FROM per_pfile",
262 table.c_str(), joinCond.c_str(), uploadFilter.c_str());
264 std::map<int, int> resultTypes;
269 int decId = std::stoi(row[0]);
270 int pfileId = std::stoi(row[1]);
271 int decType = std::stoi(row[2]);
273 auto it = result.find(pfileId);
274 if (it == result.end()) {
275 result[pfileId] = decId;
276 resultTypes[pfileId] = decType;
279 LOG_NOTICE(
"Reuser: conflicting decisions for pfile %d,"
280 " applying stronger decision type %d over %d.",
281 pfileId, decType, resultTypes[pfileId]);
282 result[pfileId] = decId;
283 resultTypes[pfileId] = decType;
290 std::map<int, std::vector<int>>
292 int uploadId,
const std::vector<int>& pfileIds)
294 std::map<int, std::vector<int>> result;
295 if (pfileIds.empty())
return result;
302 for (
size_t i = 0; i < pfileIds.size(); ++i)
304 if (i > 0) arr +=
",";
305 arr += std::to_string(pfileIds[i]);
308 bool needsUploadFilter =
309 (table ==
"uploadtree" || table ==
"uploadtree_a");
314 "SELECT uploadtree_pk, pfile_fk FROM %s"
315 " WHERE upload_fk=%d AND pfile_fk=ANY('{%s}'::int[])",
316 table.c_str(), uploadId, arr.c_str())
318 "SELECT uploadtree_pk, pfile_fk FROM %s"
319 " WHERE pfile_fk=ANY('{%s}'::int[])",
320 table.c_str(), arr.c_str());
325 int pk = std::stoi(row[0]);
326 int pfileId = std::stoi(row[1]);
327 if (pk > 0 && pfileId > 0)
328 result[pfileId].push_back(pk);
336 int uploadId,
int uploadTreeId,
int userId,
int groupId,
337 int licenseId,
bool removed,
int type,
338 const std::string& reportInfo,
const std::string& comment,
339 const std::string& ack,
int jobId)
345 const char* removedStr = removed ?
"t" :
"f";
352 table =
"uploadtree";
355 "INSERT INTO clearing_decision"
356 " (uploadtree_fk, pfile_fk, user_fk, group_fk, decision_type, scope)"
358 " (SELECT pfile_fk FROM %s WHERE uploadtree_pk=%d),"
360 uploadTreeId, table.c_str(), uploadTreeId, userId, groupId);
364 "reuserInsertClearingEvent",
365 "INSERT INTO clearing_event"
366 " (uploadtree_fk, user_fk, group_fk, type_fk, rf_fk,"
367 " removed, reportinfo, comment, acknowledgement)"
368 " VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9)"
369 " RETURNING clearing_event_pk",
370 int,
int,
int,
int,
int,
char*,
char*,
char*,
char*),
371 uploadTreeId, userId, groupId, type, licenseId,
372 removedStr, safeReport.c_str(), safeComment.c_str(), safeAck.c_str());
375 return std::stoi(qr.
getRow(0)[0]);
381 "reuserInsertClearingEventWithJob",
382 "INSERT INTO clearing_event"
383 " (uploadtree_fk, user_fk, group_fk, type_fk, rf_fk,"
384 " removed, reportinfo, comment, acknowledgement, job_fk)"
385 " VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10)"
386 " RETURNING clearing_event_pk",
387 int,
int,
int,
int,
int,
char*,
char*,
char*,
char*,
int),
388 uploadTreeId, userId, groupId, type, licenseId,
389 removedStr, safeReport.c_str(), safeComment.c_str(), safeAck.c_str(),
393 return std::stoi(qr.
getRow(0)[0]);
398 int uploadId,
int uploadTreeId,
int userId,
int groupId,
399 int decType,
int scope,
const std::vector<int>& eventIds)
401 if (eventIds.empty())
return 0;
403 if (!
begin())
return 0;
408 "reuserRemoveWipDecision",
409 "DELETE FROM clearing_decision"
410 " WHERE uploadtree_fk=$1 AND group_fk=$2 AND decision_type=0",
412 uploadTreeId, groupId);
414 if (!rRem) {
rollback();
return 0; }
418 table =
"uploadtree";
421 "INSERT INTO clearing_decision"
422 " (uploadtree_fk, pfile_fk, user_fk, group_fk, decision_type, scope)"
424 " (SELECT pfile_fk FROM %s WHERE uploadtree_pk=%d),"
426 " RETURNING clearing_decision_pk",
427 uploadTreeId, table.c_str(), uploadTreeId, userId, groupId, decType, scope);
430 int decisionPk = std::stoi(rIns.
getRow(0)[0]);
437 "reuserInsertClearingDecisionEvent",
438 "INSERT INTO clearing_decision_event"
439 " (clearing_decision_fk, clearing_event_fk) VALUES($1,$2)",
442 for (
int evPk : eventIds)
446 LOG_WARNING(
"Reuser: failed to link clearing_event %d to"
447 " clearing_decision %d, continuing.", evPk, decisionPk);
455 int uploadId,
int newItemUploadTreePk,
int userId,
int groupId,
456 int originalDecisionPk)
461 "reuserGetDecisionMeta",
462 "SELECT decision_type, scope FROM clearing_decision"
463 " WHERE clearing_decision_pk=$1",
468 int decType = std::stoi(rMeta.
getRow(0)[0]);
469 int scope = std::stoi(rMeta.
getRow(0)[1]);
474 "reuserGetEventsForDecision",
475 "SELECT ce.rf_fk, ce.removed,"
476 " ce.reportinfo, ce.comment, ce.acknowledgement"
477 " FROM clearing_event ce"
478 " INNER JOIN clearing_decision_event cde"
479 " ON cde.clearing_event_fk = ce.clearing_event_pk"
480 " WHERE cde.clearing_decision_fk=$1"
481 " ORDER BY ce.clearing_event_pk ASC",
485 if (!rEvents)
return 0;
488 std::vector<int> newEventIds;
492 auto row = rEvents.
getRow(i);
493 int rfFk = std::stoi(row[0]);
494 bool isRemoved = (row[1] ==
"t" || row[1] ==
"true");
499 rfFk, isRemoved, evType,
500 row[2], row[3], row[4],
jobId);
502 newEventIds.push_back(evPk);
505 if (newEventIds.empty())
return 0;
507 decType, scope, newEventIds);
513 int arsId,
bool success)
516 "reuser_ars",
nullptr, success ? 1 : 0);
522 int uploadId,
int reusedUploadId,
523 int groupId,
int reusedGroupId,
int userId)
526 if (reusedMap.empty())
return true;
531 std::vector<int> toImport;
532 for (
const auto& kv : reusedMap)
533 if (currentMap.find(kv.first) == currentMap.end())
534 toImport.push_back(kv.first);
536 if (toImport.empty())
return true;
538 constexpr
size_t chunkSize = 100;
539 for (
size_t i = 0; i < toImport.size(); i += chunkSize)
541 size_t end =
std::min(i + chunkSize, toImport.size());
542 std::vector<int> chunk(toImport.begin() + i, toImport.begin() + end);
545 for (
const auto& entry : pkMap)
547 int originalDecision = reusedMap.at(entry.first);
548 for (
int uploadtreePk : entry.second)
551 uploadId, uploadtreePk, userId, groupId, originalDecision);
561 int uploadId,
int reusedUploadId,
562 int groupId,
int reusedGroupId,
int userId)
565 if (reusedMap.empty())
return true;
569 std::vector<int> toImport;
570 for (
const auto& kv : reusedMap)
571 if (currentMap.find(kv.first) == currentMap.end())
572 toImport.push_back(kv.first);
574 if (toImport.empty())
return true;
581 bool reusedNeedsFilter = (tableReused ==
"uploadtree" || tableReused ==
"uploadtree_a");
582 bool targetNeedsFilter = (tableTarget ==
"uploadtree" || tableTarget ==
"uploadtree_a");
584 std::string reusedFilter = reusedNeedsFilter
585 ?
" AND ur.upload_fk=" + std::to_string(reusedUploadId) :
"";
586 std::string targetFilter = targetNeedsFilter
587 ?
" AND ut.upload_fk=" + std::to_string(uploadId) :
"";
589 for (
int pfileFk : toImport)
591 int originalDecision = reusedMap.at(pfileFk);
594 if (reusedPath.empty())
continue;
598 "SELECT ut.uploadtree_pk, ut.pfile_fk"
600 " WHERE ur.pfile_fk=%d%s"
601 " AND ut.ufile_name=ur.ufile_name%s",
602 tableReused.c_str(), tableTarget.c_str(),
603 pfileFk, reusedFilter.c_str(),
604 targetFilter.c_str());
609 int newItemPk = std::stoi(row[0]);
610 int newPfileFk = std::stoi(row[1]);
611 if (newItemPk <= 0 || newPfileFk <= 0)
continue;
614 if (newPath.empty())
continue;
616 int diffCount = diffLineCount(reusedPath, newPath);
617 if (diffCount < 0)
return false;
621 uploadId, newItemPk, userId, groupId, originalDecision);
631 int uploadId,
int groupId,
int reusedUploadId,
int reusedGroupId)
635 "reuserGetReusedMainLicenses",
636 "SELECT rf_fk FROM upload_clearing_license"
637 " WHERE upload_fk=$1 AND group_fk=$2",
639 reusedUploadId, reusedGroupId);
641 std::set<int> reusedSet;
643 reusedSet.insert(std::stoi(r1.
getRow(i)[0]));
645 if (reusedSet.empty())
return true;
649 "reuserGetTargetMainLicenses",
650 "SELECT rf_fk FROM upload_clearing_license"
651 " WHERE upload_fk=$1 AND group_fk=$2",
655 std::set<int> existingSet;
657 existingSet.insert(std::stoi(r2.
getRow(i)[0]));
659 for (
int rf : reusedSet)
661 if (existingSet.count(rf))
continue;
664 "reuserInsertMainLicense",
665 "INSERT INTO upload_clearing_license (upload_fk, group_fk, rf_fk)"
666 " VALUES ($1,$2,$3)",
668 uploadId, groupId, rf);
669 if (!rIns)
return false;
675 int uploadId,
int reusedUploadId)
680 "reuserConfSettingsCheck",
681 "SELECT 1 FROM report_info WHERE upload_fk=$1 LIMIT 1",
685 if (!rCheck || rCheck.
getRowCount() == 0)
return true;
687 if (!
begin())
return false;
692 "reuserConfSettingsDelete",
693 "DELETE FROM report_info WHERE upload_fk=$1",
697 if (!rDel) {
rollback();
return false; }
702 "SELECT string_agg(quote_ident(column_name), ',')"
703 " FROM information_schema.columns"
704 " WHERE table_schema = current_schema()"
705 " AND table_name = 'report_info'"
706 " AND column_name != 'ri_pk'"
707 " AND column_name != 'upload_fk'");
710 std::string cols = rCols.
getRow(0)[0];
711 if (cols.empty()) {
rollback();
return false; }
715 "INSERT INTO report_info(upload_fk, %s)"
716 " SELECT %d, %s FROM report_info WHERE upload_fk=%d",
717 cols.c_str(), uploadId, cols.c_str(), reusedUploadId);
719 if (!rCopy) {
rollback();
return false; }
726 int uploadId,
int reusedUploadId,
int userId)
728 const std::string agentName =
"copyright";
733 "reuserCopyrightTargetAgentId",
734 "SELECT agent_pk AS agent_id FROM agent"
735 " LEFT JOIN copyright_ars ON agent_fk=agent_pk"
736 " WHERE agent_name=$2 AND agent_enabled"
737 " AND upload_fk=$1 AND ars_success"
738 " ORDER BY agent_pk DESC LIMIT 1",
740 uploadId, agentName.c_str());
742 if (!rAgentT || rAgentT.
getRowCount() == 0)
return true;
743 int targetAgentId = std::stoi(rAgentT.
getRow(0)[0]);
747 "reuserCopyrightReusedAgentId",
748 "SELECT agent_pk AS agent_id FROM agent"
749 " LEFT JOIN copyright_ars ON agent_fk=agent_pk"
750 " WHERE agent_name=$2 AND agent_enabled"
751 " AND upload_fk=$1 AND ars_success"
752 " ORDER BY agent_pk DESC LIMIT 1",
754 reusedUploadId, agentName.c_str());
756 if (!rAgentR || rAgentR.
getRowCount() == 0)
return true;
757 int reusedAgentId = std::stoi(rAgentR.
getRow(0)[0]);
763 bool needsUploadFilter = (table ==
"uploadtree" || table ==
"uploadtree_a");
764 std::string uploadFilter = needsUploadFilter
765 ?
" AND UT.upload_fk = " + std::to_string(uploadId) :
"";
768 "SELECT DISTINCT ON (C.copyright_pk, UT.uploadtree_pk)"
769 " C.copyright_pk, UT.uploadtree_pk, UT.upload_fk,"
770 " (CASE WHEN (CE.content IS NULL OR CE.content = '')"
771 " THEN C.content ELSE CE.content END) AS content,"
772 " (CASE WHEN (CE.hash IS NULL OR CE.hash = '')"
773 " THEN C.hash ELSE CE.hash END) AS hash"
775 " INNER JOIN %s UT ON C.pfile_fk = UT.pfile_fk%s"
776 " LEFT JOIN copyright_event CE"
777 " ON CE.copyright_fk = C.copyright_pk"
778 " AND CE.upload_fk = %d"
779 " AND CE.uploadtree_fk = UT.uploadtree_pk"
780 " WHERE C.content IS NOT NULL AND C.content <> ''"
781 " AND (CE.is_enabled IS NULL OR CE.is_enabled = 'true')"
782 " AND C.agent_fk = %d"
783 " ORDER BY C.copyright_pk, UT.uploadtree_pk, content DESC",
784 table.c_str(), uploadFilter.c_str(), uploadId, targetAgentId);
788 using Row3 = std::array<int, 3>;
789 std::map<std::string, std::vector<Row3>> allMap;
792 auto row = rAll.
getRow(i);
793 std::string hash = row[4];
795 allMap[hash].push_back(Row3{std::stoi(row[0]), std::stoi(row[1]),
798 if (allMap.empty())
return true;
804 "reuserGetReusedCopyrightEvents",
805 "SELECT C.copyright_pk, CE.is_enabled, C.hash,"
806 " CE.content AS contentedited"
807 " FROM copyright_event CE"
808 " INNER JOIN copyright C ON C.copyright_pk = CE.copyright_fk"
809 " WHERE CE.upload_fk=$1 AND CE.scope=$3 AND C.agent_fk=$2",
811 reusedUploadId, reusedAgentId, 1 );
815 auto rRow = rReused.
getRow(i);
816 std::string hash = rRow[2];
817 if (hash.empty())
continue;
819 auto it = allMap.find(hash);
820 if (it == allMap.end() || it->second.empty())
continue;
822 Row3 entry = it->second.back();
823 it->second.pop_back();
824 int copyrightPk = entry[0];
825 int uploadtreePk = entry[1];
826 int uploadFk = entry[2];
828 const std::string& contentEdited = rRow[3];
833 "reuserCopyrightEventExists",
835 " SELECT 1 FROM copyright_event"
836 " WHERE copyright_fk=$1 AND upload_fk=$2 AND uploadtree_fk=$3"
839 copyrightPk, uploadFk, uploadtreePk);
841 bool eventExists = rExists && rExists.
getRowCount() > 0
842 && std::stoi(rExists.
getRow(0)[0]) != 0;
853 "reuserCopyrightEventDisableUpdate",
854 "UPDATE copyright_event SET scope=$4, is_enabled=false"
855 " WHERE upload_fk=$1 AND copyright_fk=$2 AND uploadtree_fk=$3",
857 uploadFk, copyrightPk, uploadtreePk, 1)
860 "reuserCopyrightEventDisableInsert",
861 "INSERT INTO copyright_event"
862 " (upload_fk, copyright_fk, uploadtree_fk, is_enabled, scope)"
863 " VALUES($1,$2,$3,'f',$4)",
865 uploadFk, copyrightPk, uploadtreePk, 1);
867 LOG_WARNING(
"Reuser: failed to disable copyright_event"
868 " (copyright_fk=%d, uploadtree_fk=%d), continuing.",
869 copyrightPk, uploadtreePk);
877 "reuserCopyrightEventUpdateContent",
878 "UPDATE copyright_event SET upload_fk=$1, content=$4,"
880 " WHERE copyright_fk=$2 AND uploadtree_fk=$3",
881 int,
int,
int,
char*),
882 uploadFk, copyrightPk, uploadtreePk, contentEdited.c_str())
885 "reuserCopyrightEventInsertContent",
886 "INSERT INTO copyright_event"
887 " (upload_fk, uploadtree_fk, copyright_fk,"
888 " is_enabled, content, hash)"
889 " VALUES($1,$3,$2,'true',$4,md5($4))",
890 int,
int,
int,
char*),
891 uploadFk, copyrightPk, uploadtreePk, contentEdited.c_str());
893 LOG_WARNING(
"Reuser: failed to update copyright_event content"
894 " (copyright_fk=%d, uploadtree_fk=%d), continuing.",
895 copyrightPk, uploadtreePk);
904 std::vector<int> bulkIds;
908 "reuserGetPreviousBulkIds",
909 "SELECT jq_args FROM upload_reuse, jobqueue, job"
910 " WHERE upload_fk=$1 AND group_fk=$2"
911 " AND (reuse_mode & 8) = 8"
912 " AND EXISTS(SELECT * FROM group_user_member gum WHERE gum.group_fk=upload_reuse.group_fk AND gum.user_fk=$3)"
913 " AND jq_type=$4 AND jq_job_fk=job_pk"
914 " AND job_upload_fk=reused_upload_fk AND job_group_fk=reused_group_fk"
916 int,
int,
int,
char*),
917 uploadId, groupId, userId, (
char*)
"monkbulk");
919 std::set<int> seenBulkIds;
922 std::string jqArgs = qr.
getRow(i)[0];
923 std::stringstream ss(jqArgs);
925 while (std::getline(ss, line,
'\n'))
930 int bulkId = std::stoi(line);
931 if (seenBulkIds.insert(bulkId).second)
933 bulkIds.push_back(bulkId);
946 if (bulkIds.empty()) {
953 for (
int bulkId : bulkIds) {
969 if (timeInSec > maxTime) {
971 }
else if (timeInSec < minTime) {
988 int nTopItem = bounds.uploadtree_pk;
992 "reuserGetBulkUploadtree",
993 "SELECT uploadtree_fk FROM license_ref_bulk WHERE lrb_pk=$1",
997 LOG_WARNING(
"Reuser: no license_ref_bulk row found for lrb_pk %d", bulkId);
1000 int bulkUploadtreeFk = 0;
1002 bulkUploadtreeFk = std::stoi(qrBulk.
getRow(0)[0]);
1004 LOG_WARNING(
"Reuser: invalid uploadtree_fk for lrb_pk %d", bulkId);
1009 std::string ufileName;
1014 "reuserGetUploadtreeEntry1",
1015 "SELECT upload_fk, ufile_name, ufile_mode FROM uploadtree WHERE uploadtree_pk=$1",
1019 pUID = std::stoi(qrEntry.
getRow(0)[0]);
1020 ufileName = qrEntry.
getRow(0)[1];
1021 ufileMode = std::stoi(qrEntry.
getRow(0)[2]);
1025 "reuserGetUploadtreeEntry2",
1026 "SELECT upload_fk, ufile_name, ufile_mode FROM uploadtree_a WHERE uploadtree_pk=$1",
1030 pUID = std::stoi(qrEntry2.
getRow(0)[0]);
1031 ufileName = qrEntry2.
getRow(0)[1];
1032 ufileMode = std::stoi(qrEntry2.
getRow(0)[2]);
1034 LOG_WARNING(
"Reuser: no uploadtree entry found for lrb_pk %d uploadtree_fk %d",
1035 bulkId, bulkUploadtreeFk);
1042 LOG_WARNING(
"Reuser: getParentItemBounds failed for upload %d", pUID);
1045 int pTopItem = pBounds.uploadtree_pk;
1048 if (pTopItem == bulkUploadtreeFk) {
1054 "reuserGetMatchingEntry1",
1055 "SELECT uploadtree_pk FROM uploadtree WHERE upload_fk=$1 AND ufile_name=$2 AND ufile_mode=$3",
1057 uploadId, ufileName.c_str(), ufileMode);
1060 topItem = std::stoi(qrMatch.
getRow(0)[0]);
1062 LOG_WARNING(
"Reuser: invalid matching uploadtree entry for upload %d", uploadId);
1068 "reuserGetMatchingEntry2",
1069 "SELECT uploadtree_pk FROM uploadtree_a WHERE upload_fk=$1 AND ufile_name=$2 AND ufile_mode=$3",
1071 uploadId, ufileName.c_str(), ufileMode);
1074 topItem = std::stoi(qrMatch2.
getRow(0)[0]);
1076 LOG_WARNING(
"Reuser: invalid matching uploadtree_a entry for upload %d", uploadId);
1082 LOG_WARNING(
"Reuser: no matching uploadtree entry in new upload %d for"
1083 " ufile_name='%s' ufile_mode=%d", uploadId, ufileName.c_str(), ufileMode);
1090 "INSERT INTO license_ref_bulk (user_fk,group_fk,rf_text,upload_fk,uploadtree_fk,ignore_irrelevant,bulk_delimiters,scan_findings) "
1091 "SELECT $1 AS user_fk, $2 AS group_fk,rf_text,$3 AS upload_fk, $4 as uploadtree_fk, ignore_irrelevant, bulk_delimiters, scan_findings "
1092 "FROM license_ref_bulk WHERE lrb_pk=$5 RETURNING lrb_pk",
1093 int,
int,
int,
int,
int),
1094 userId, groupId, uploadId, topItem, bulkId);
1096 LOG_WARNING(
"Reuser: clone license_ref_bulk failed for lrb_pk %d", bulkId);
1101 newBulkId = std::stoi(qrClone.
getRow(0)[0]);
1103 LOG_WARNING(
"Reuser: invalid newBulkId from RETURNING for lrb_pk %d", bulkId);
1109 "reuserCloneBulkLic",
1110 "INSERT INTO license_set_bulk (lrb_fk, rf_fk, removing, comment, reportinfo, acknowledgement) "
1111 "SELECT $1 as lrb_fk, rf_fk, removing, comment, reportinfo, acknowledgement FROM license_set_bulk WHERE lrb_fk=$2",
1117 "reuserGetUploadFilename",
1118 "SELECT upload_filename FROM upload WHERE upload_pk=$1",
1122 LOG_WARNING(
"Reuser: no upload row found for upload_pk %d", uploadId);
1125 std::string uploadName = qrUpload.
getRow(0)[0];
1130 "INSERT INTO job (job_user_fk, job_group_fk, job_queued, job_priority, job_name, job_upload_fk)"
1131 " VALUES ($1, $2, now(), 0, $3, $4) RETURNING job_pk",
1132 int,
int,
char*,
int),
1133 userId, groupId, uploadName.c_str(), uploadId);
1135 LOG_WARNING(
"Reuser: failed to create job for upload %d", uploadId);
1140 jobPk = std::stoi(qrJob.
getRow(0)[0]);
1142 LOG_WARNING(
"Reuser: invalid job_pk from RETURNING for upload %d", uploadId);
1146 std::string newBulkIdStr = std::to_string(newBulkId);
1149 "reuserScheduleMonkBulk",
1150 "INSERT INTO jobqueue (jq_job_fk, jq_type, jq_args, jq_runonpfile, jq_starttime, jq_endtime, jq_end_bits, jq_host, jq_cmd_args)"
1151 " VALUES ($1, 'monkbulk', $2, NULL, NULL, NULL, 0, NULL, NULL) RETURNING jq_pk",
1153 jobPk, newBulkIdStr.c_str());
1155 LOG_WARNING(
"Reuser: failed to schedule monkbulk for job %d", jobPk);
1160 monkJqPk = std::stoi(qrMonk.
getRow(0)[0]);
1162 LOG_WARNING(
"Reuser: invalid monkbulk jq_pk for job %d", jobPk);
1168 "reuserCheckAdj2Nest",
1169 "SELECT jq_pk FROM jobqueue, job WHERE job_pk=jq_job_fk"
1170 " AND jq_type='adj2nest' AND job_upload_fk=$1",
1173 int adj2nestJqPk = 0;
1175 adj2nestJqPk = std::stoi(qrAdjCheck.
getRow(0)[0]);
1177 std::string uploadIdStr = std::to_string(uploadId);
1180 "reuserScheduleAdj2Nest",
1181 "INSERT INTO jobqueue (jq_job_fk, jq_type, jq_args, jq_runonpfile, jq_starttime, jq_endtime, jq_end_bits, jq_host, jq_cmd_args)"
1182 " VALUES ($1, 'adj2nest', $2, NULL, NULL, NULL, 0, NULL, NULL) RETURNING jq_pk",
1184 jobPk, uploadIdStr.c_str());
1186 adj2nestJqPk = std::stoi(qrAdj.
getRow(0)[0]);
1190 std::string uploadIdStr = std::to_string(uploadId);
1193 LOG_WARNING(
"Reuser: failed to begin transaction for scheduling upload %d", uploadId);
1199 "reuserScheduleDecider",
1200 "INSERT INTO jobqueue (jq_job_fk, jq_type, jq_args, jq_runonpfile, jq_starttime, jq_endtime, jq_end_bits, jq_host, jq_cmd_args)"
1201 " VALUES ($1, 'deciderjob', $2, NULL, NULL, NULL, 0, NULL, NULL) RETURNING jq_pk",
1203 jobPk, uploadIdStr.c_str());
1205 LOG_WARNING(
"Reuser: failed to schedule deciderjob for job %d", jobPk);
1209 int deciderJqPk = 0;
1211 deciderJqPk = std::stoi(qrDecider.
getRow(0)[0]);
1213 LOG_WARNING(
"Reuser: invalid deciderjob jq_pk for job %d", jobPk);
1220 "reuserAddDependencyMonk",
1221 "INSERT INTO jobdepends (jdep_jq_fk, jdep_jq_depends_fk) VALUES ($1, $2)",
1223 deciderJqPk, monkJqPk);
1225 LOG_WARNING(
"Reuser: failed to add monkbulk dependency for job %d", jobPk);
1230 if (adj2nestJqPk > 0) {
1233 "reuserAddDependencyAdj",
1234 "INSERT INTO jobdepends (jdep_jq_fk, jdep_jq_depends_fk) VALUES ($1, $2)",
1236 deciderJqPk, adj2nestJqPk);
1238 LOG_WARNING(
"Reuser: failed to add adj2nest dependency for job %d", jobPk);
1245 LOG_WARNING(
"Reuser: failed to commit scheduling for job %d", jobPk);
1258 "reuserIsJobQueueRunning",
1259 "SELECT jq_end_bits FROM jobqueue WHERE jq_pk = $1",
1268 int endBits = std::stoi(qr.
getRow(0)[0]);
1269 return !(endBits == 1 || endBits == 2);
1279 "reuserGetJobFkFromJqPk",
1280 "SELECT jq_job_fk FROM jobqueue WHERE jq_pk = $1",
1289 valJob = qrJob.
getRow(0)[0];
1290 if (!valJob.empty()) {
1291 jobPk = std::stoi(valJob);
1293 }
catch (
const std::exception& e) {
1294 LOG_WARNING(
"Reuser: failed to parse job FK for jqPk %d (val='%s'): %s",
1295 jqPk, valJob.c_str(), e.what());
1298 LOG_WARNING(
"Reuser: unknown error parsing job FK for jqPk %d (val='%s')",
1299 jqPk, valJob.c_str());
1305 "reuserGetUnunpackProcessed",
1306 "SELECT jq_itemsprocessed FROM jobqueue WHERE jq_type = 'ununpack' AND jq_end_bits = 1 AND jq_job_fk = $1",
1312 int ununpackProcessed = 0;
1313 std::string valUnpack;
1315 valUnpack = qrUnpack.
getRow(0)[0];
1316 if (!valUnpack.empty()) {
1317 ununpackProcessed = std::stoi(valUnpack);
1319 }
catch (
const std::exception& e) {
1320 LOG_WARNING(
"Reuser: failed to parse ununpack processed count (val='%s'): %s",
1321 valUnpack.c_str(), e.what());
1324 LOG_WARNING(
"Reuser: unknown error parsing ununpack processed count (val='%s')",
1329 if (ununpackProcessed <= 0) {
1335 "reuserGetEstimatedTimeAgents",
1336 "SELECT jq_itemsprocessed, EXTRACT(EPOCH FROM (now() - jq_starttime)) AS burn_time FROM jobqueue "
1337 "WHERE jq_type <> 'ununpack' AND jq_type <> 'reportgen' AND jq_type <> 'decider' AND jq_type <> 'softwareHeritage' "
1338 " AND jq_job_fk = $1 AND jq_endtime IS NULL AND jq_starttime IS NOT NULL",
1342 double maxCompletionTime = 0.0;
1343 bool foundEstimate =
false;
1345 for (
int i = 0; i < qrAgents.
getRowCount(); ++i) {
1346 int jqItemsProcessed = 0;
1347 double burnTime = 0.0;
1348 std::string val0, val1;
1350 val0 = qrAgents.
getRow(i)[0];
1351 val1 = qrAgents.
getRow(i)[1];
1352 if (!val0.empty()) {
1353 jqItemsProcessed = std::stoi(val0);
1355 if (!val1.empty()) {
1356 burnTime = std::stod(val1);
1358 }
catch (
const std::exception& e) {
1359 LOG_WARNING(
"Reuser: failed to parse agent jobqueue row (val0='%s', val1='%s'): %s",
1360 val0.c_str(), val1.c_str(), e.what());
1363 LOG_WARNING(
"Reuser: unknown error parsing agent jobqueue row (val0='%s', val1='%s')",
1364 val0.c_str(), val1.c_str());
1368 if (burnTime > 0.0) {
1369 double filesPerSec =
static_cast<double>(jqItemsProcessed) / burnTime;
1370 if (filesPerSec > 0.0) {
1371 double timeOfCompletion =
static_cast<double>(ununpackProcessed - jqItemsProcessed) / filesPerSec;
1372 if (timeOfCompletion > maxCompletionTime) {
1373 maxCompletionTime = timeOfCompletion;
1375 foundEstimate =
true;
1380 if (!foundEstimate) {
1384 return static_cast<int>(maxCompletionTime + 0.5);
1391 if (!host || !port)
return;
1393 struct addrinfo hints, *servs, *curr =
nullptr;
1394 memset(&hints, 0,
sizeof(hints));
1395 hints.ai_family = AF_UNSPEC;
1396 hints.ai_socktype = SOCK_STREAM;
1397 if (getaddrinfo(host, port, &hints, &servs) != 0) {
1402 for (curr = servs; curr !=
nullptr; curr = curr->ai_next) {
1403 fd = socket(curr->ai_family, hints.ai_socktype, curr->ai_protocol);
1404 if (fd < 0)
continue;
1405 if (connect(fd, curr->ai_addr, curr->ai_addrlen) == 0) {
1411 freeaddrinfo(servs);
1414 if (write(fd,
"database", 8) < 0) {
Database handler for the reuser agent.
virtual bool processUploadReuse(int uploadId, int reusedUploadId, int groupId, int reusedGroupId, int userId)
virtual void notifySchedulerOfDatabaseChange()
virtual bool getParentItemBounds(int uploadId, ItemTreeBounds &out)
Fetch the parent item bounds for a given upload.
virtual int createCopyOfClearingDecision(int uploadId, int newItemUploadTreePk, int userId, int groupId, int originalDecisionPk)
Copy an existing clearing decision to a new uploadtree item.
virtual int createDecisionFromEvents(int uploadId, int uploadTreeId, int userId, int groupId, int decType, int scope, const std::vector< int > &eventIds)
Create a clearing_decision linked to eventIds.
virtual bool reuseMainLicense(int uploadId, int groupId, int reusedUploadId, int reusedGroupId)
virtual bool processBulkReuser(int uploadId, int groupId, int userId)
virtual int getEstimatedTime(int jqPk)
virtual std::vector< int > getPreviousBulkIds(int uploadId, int groupId, int userId)
virtual ReuserDatabaseHandler spawn() const
virtual std::map< int, int > getClearingDecisionMapByPfile(int uploadId, int groupId)
Build a pfile_fk to clearing_decision_pk map for uploadId.
static int getDecisionTypePriority(int decisionType)
Priority for decision types during reuse conflict resolution.
virtual int rerunBulkAndDeciderOnUpload(int uploadId, int groupId, int userId, int bulkId)
virtual bool isJobQueueRunning(int jqPk)
virtual bool processEnhancedUploadReuse(int uploadId, int reusedUploadId, int groupId, int reusedGroupId, int userId)
static bool isValidIdentifier(const std::string &s)
Validate that s contains only characters safe for SQL identifiers.
std::string getRepoPathOfPfile(int pfileId)
virtual int insertClearingEvent(int uploadId, int uploadTreeId, int userId, int groupId, int licenseId, bool removed, int type, const std::string &reportInfo, const std::string &comment, const std::string &ack, int jobId)
Insert a new clearing event and return its primary key (0 on error).
static std::string replaceUnicodeControlChars(const std::string &input)
Strip Unicode control characters (C0, C1, DEL) from input.
virtual bool reuseCopyrights(int uploadId, int reusedUploadId, int userId)
virtual int writeArsRecord(int agentId, int uploadId, int arsId=0, bool success=false)
Write (insert or update) an ARS record.
virtual std::vector< ReuseTriple > getReusedUploads(int uploadId, int groupId)
Return the list of uploads that should be reused for uploadId.
virtual bool reuseConfSettings(int uploadId, int reusedUploadId)
virtual std::map< int, std::vector< int > > getUploadTreePksForPfiles(int uploadId, const std::vector< int > &pfileIds)
For a set of pfile ids, return a map pfile_fk to [uploadtree_pk].
Database handler for agents.
std::string queryUploadTreeTableName(int uploadId)
Get the upload tree table name for a given upload id.
bool commit() const
COMMIT a transaction block in DB.
bool begin() const
BEGIN a transaction block in DB.
char * getPFileNameForFileId(unsigned long pfileId) const
Get the file name of a give pfile id.
DbManager dbManager
DbManager to use.
bool rollback() const
ROLLBACK a transaction block in DB.
QueryResult execPrepared(fo_dbManager_PreparedStatement *stmt,...) const
Execute a prepared statement with new parameters.
QueryResult queryPrintf(const char *queryFormat,...) const
Execute a query in printf format.
PGconn * getConnection() const
fo_dbManager * getStruct_dbManager() const
std::vector< std::string > getRow(int i) const
int s
The socket that the CLI will use to communicate.
FUNCTION int min(int user_perm, int permExternal)
Get the minimum permission level required.
FUNCTION int fo_WriteARS(PGconn *pgConn, int ars_pk, int upload_pk, int agent_pk, const char *tableName, const char *ars_status, int ars_success)
Write ars record.
char * fo_RepMkPath(const char *Type, char *Filename)
Given a filename, construct the full path to the file.
void fo_scheduler_heart(int i)
This function must be called by agents to let the scheduler know they are alive and how many items th...
char * fo_sysconfig(const char *sectionname, const char *variablename)
gets a system configuration variable from the configuration data.
int jobId
The id of the job.
int fo_scheduler_jobId()
Gets the id of the job that the agent is running.
fo_dbManager * dbManager
fo_dbManager object
fo namespace holds the FOSSology library functions.
bool stringToBool(const char *string)
Bounds of an item within an uploadtree table.