FOSSology  4.7.1
Open Source License Compliance by Open Source Software
ReuserDatabaseHandler.cc
1 /*
2  SPDX-License-Identifier: GPL-2.0-only
3  Author: Dietmar Helmut Leher <helmut.leher.ext@vaillant-group.com>
4  SPDX-FileCopyrightText: © 2026 Vaillant GmbH
5 */
6 
7 #include "ReuserDatabaseHandler.hpp"
8 
9 #include <algorithm>
10 #include <cstdio>
11 #include <cstdlib>
12 #include <cstring>
13 #include <set>
14 #include <sstream>
15 #include <sys/wait.h>
16 #include <sys/socket.h>
17 #include <netdb.h>
18 #include <unistd.h>
19 
20 #include <unicode/unistr.h>
21 
22 extern "C" {
23 #include "libfossagent.h"
24 }
25 
26 using namespace fo;
27 
28 // Construction
29 
30 ReuserDatabaseHandler::ReuserDatabaseHandler(DbManager dbManager)
32 {
33 }
34 
36 {
38 }
39 
40 // Private helpers
41 
42 /* Mirror of DecisionTypes.php; keep in sync if the PHP enum changes.
43  * Types 1 and 2 do not exist in that enum; default covers them. */
44 namespace {
45  constexpr int DT_WIP = 0;
46  constexpr int DT_TO_BE_DISCUSSED = 3;
47  constexpr int DT_IRRELEVANT = 4;
48  constexpr int DT_IDENTIFIED = 5;
49  constexpr int DT_DO_NOT_USE = 6;
50  constexpr int DT_NON_FUNCTIONAL = 7;
51 }
52 
54 {
55  switch (decisionType)
56  {
57  case DT_IDENTIFIED: return 5;
58  case DT_DO_NOT_USE: return 4;
59  case DT_NON_FUNCTIONAL: return 3;
60  case DT_IRRELEVANT: return 2;
61  case DT_TO_BE_DISCUSSED: return 1;
62  default: return 0;
63  }
64 }
65 
67 {
68  if (s.empty()) return false;
69  for (char c : s)
70  if (!((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') ||
71  (c >= '0' && c <= '9') || c == '_'))
72  return false;
73  return true;
74 }
75 
77  const std::string& input)
78 {
79  icu::UnicodeString us = icu::UnicodeString::fromUTF8(input);
80  icu::UnicodeString result;
81  for (int32_t i = 0; i < us.length(); ++i)
82  {
83  UChar32 cp = us.char32At(i);
84  if (cp > 0xFFFF) ++i; // surrogate pair: char32At already consumed 2 units
85  bool isControl = (cp <= 0x08)
86  || (cp == 0x0B)
87  || (cp == 0x0C)
88  || (cp >= 0x0E && cp <= 0x1F)
89  || (cp >= 0x7F && cp <= 0x9F);
90  if (!isControl)
91  result.append(cp);
92  }
93  std::string out;
94  result.toUTF8String(out);
95  return out;
96 }
97 
98 std::string ReuserDatabaseHandler::shellEscape(const std::string& s)
99 {
100  std::string r = "'";
101  for (char c : s)
102  r += (c == '\'') ? std::string("'\\''") : std::string(1, c);
103  r += "'";
104  return r;
105 }
106 
107 int ReuserDatabaseHandler::diffLineCount(const std::string& a,
108  const std::string& b)
109 {
110  if (a.empty() || b.empty()) return -1;
111 
112  // Run diff directly (no pipeline) so pclose() returns diff's own exit code.
113  // Redirect diff's stderr to suppress "no such file" noise.
114  std::string cmd = "diff -- " + shellEscape(a) + " " + shellEscape(b)
115  + " 2>/dev/null";
116  FILE* pipe = popen(cmd.c_str(), "r");
117  if (!pipe) return -1;
118 
119  int lines = 0;
120  char buf[4096];
121  while (fgets(buf, sizeof(buf), pipe))
122  ++lines;
123 
124  int status = pclose(pipe);
125  // diff exit codes: 0 = identical, 1 = differences found, 2 = error.
126  if (WIFEXITED(status) && WEXITSTATUS(status) == 2)
127  return -1;
128 
129  return lines;
130 }
131 
133 {
134  char* pfileName =
135  getPFileNameForFileId(static_cast<unsigned long>(pfileId));
136  if (!pfileName) return {};
137  char* filePath = fo_RepMkPath("files", pfileName);
138  free(pfileName);
139  if (!filePath) return {};
140  std::string result(filePath);
141  free(filePath);
142  return result;
143 }
144 
145 // Upload-tree helpers
146 
148  ItemTreeBounds& out)
149 {
150  std::string table = queryUploadTreeTableName(uploadId);
151  if (!isValidIdentifier(table)) return false;
152 
153  bool needsUploadFilter =
154  (table == "uploadtree" || table == "uploadtree_a");
155 
156  QueryResult result =
157  needsUploadFilter
159  "SELECT uploadtree_pk, upload_fk, lft, rgt"
160  " FROM %s WHERE parent IS NULL AND upload_fk=%d",
161  table.c_str(), uploadId)
163  "SELECT uploadtree_pk, upload_fk, lft, rgt"
164  " FROM %s WHERE parent IS NULL",
165  table.c_str());
166 
167  if (!result || result.getRowCount() == 0) return false;
168 
169  auto row = result.getRow(0);
170  out.uploadtree_pk = std::stoi(row[0]);
171  out.uploadTreeTableName = table;
172  out.upload_fk = std::stoi(row[1]);
173  out.lft = std::stoi(row[2]);
174  out.rgt = std::stoi(row[3]);
175  return true;
176 }
177 
178 // Reuse relationship queries
179 
181  int uploadId, int groupId)
182 {
183  std::vector<ReuseTriple> result;
184 
186  fo_dbManager_PrepareStamement(dbManager.getStruct_dbManager(),
187  "reuserGetReusedUploads",
188  "SELECT reused_upload_fk, reused_group_fk, reuse_mode"
189  " FROM upload_reuse"
190  " WHERE upload_fk=$1 AND group_fk=$2"
191  " ORDER BY date_added DESC",
192  int, int),
193  uploadId, groupId);
194 
195  for (int i = 0; i < qr.getRowCount(); ++i)
196  {
197  auto row = qr.getRow(i);
198  result.push_back({std::stoi(row[0]), std::stoi(row[1]),
199  std::stoi(row[2])});
200  }
201  return result;
202 }
203 
205  int uploadId, int groupId)
206 {
207  std::map<int, int> result;
208 
209  std::string table = queryUploadTreeTableName(uploadId);
210  if (!isValidIdentifier(table)) return result;
211 
212  bool needsUploadFilter =
213  (table == "uploadtree" || table == "uploadtree_a");
214 
215  // Determine whether global (REPO) decisions should be applied.
216  bool applyGlobal = true;
218  fo_dbManager_PrepareStamement(dbManager.getStruct_dbManager(),
219  "reuserGetGlobalDecision",
220  // Cast int2 to boolean so PostgreSQL returns 't'/'f' regardless of storage
221  // format ('1'/'0' vs 'true'/'false'); stringToBool only recognises 't'/'true'.
222  "SELECT (ri_globaldecision != 0) FROM report_info WHERE upload_fk=$1",
223  int),
224  uploadId);
225  if (globalQr && globalQr.getRowCount() > 0)
226  applyGlobal = fo::stringToBool(globalQr.getRow(0)[0].c_str());
227 
228  // Omit scope=0 check to mirror PHP ClearingDao::getRelevantDecisionsCte.
229  std::string joinCond =
230  applyGlobal
231  ? "(ut.pfile_fk = cd.pfile_fk AND cd.scope = 1)"
232  " OR (ut.uploadtree_pk = cd.uploadtree_fk"
233  " AND cd.scope = 0 AND cd.group_fk = " + std::to_string(groupId) + ")"
234  : "(ut.uploadtree_pk = cd.uploadtree_fk"
235  " AND cd.group_fk = " + std::to_string(groupId) + ")";
236 
237  std::string uploadFilter =
238  needsUploadFilter
239  ? " AND ut.upload_fk = " + std::to_string(uploadId)
240  : "";
241 
242  // Inner CTE: best decision per uploadtree_pk (ITEM before REPO, newest first).
243  // Outer CTE: all rows kept for priority-based conflict resolution.
245  "WITH per_item AS ("
246  " SELECT DISTINCT ON(ut.uploadtree_pk)"
247  " cd.clearing_decision_pk AS id,"
248  " cd.pfile_fk AS pfile_id,"
249  " cd.decision_type AS dec_type"
250  " FROM clearing_decision cd"
251  " INNER JOIN %s ut ON (%s)%s"
252  " WHERE cd.decision_type != 0"
253  " ORDER BY ut.uploadtree_pk, cd.scope ASC,"
254  " cd.clearing_decision_pk DESC"
255  "),"
256  " per_pfile AS ("
257  " SELECT id, pfile_id, dec_type"
258  " FROM per_item"
259  " ORDER BY pfile_id, id DESC"
260  ")"
261  " SELECT id, pfile_id, dec_type FROM per_pfile",
262  table.c_str(), joinCond.c_str(), uploadFilter.c_str());
263 
264  std::map<int, int> resultTypes;
265 
266  for (int i = 0; i < qr.getRowCount(); ++i)
267  {
268  auto row = qr.getRow(i);
269  int decId = std::stoi(row[0]);
270  int pfileId = std::stoi(row[1]);
271  int decType = std::stoi(row[2]);
272  if (pfileId > 0) {
273  auto it = result.find(pfileId);
274  if (it == result.end()) {
275  result[pfileId] = decId;
276  resultTypes[pfileId] = decType;
277  } else if (getDecisionTypePriority(decType) >
278  getDecisionTypePriority(resultTypes[pfileId])) {
279  LOG_NOTICE("Reuser: conflicting decisions for pfile %d,"
280  " applying stronger decision type %d over %d.",
281  pfileId, decType, resultTypes[pfileId]);
282  result[pfileId] = decId;
283  resultTypes[pfileId] = decType;
284  }
285  }
286  }
287  return result;
288 }
289 
290 std::map<int, std::vector<int>>
292  int uploadId, const std::vector<int>& pfileIds)
293 {
294  std::map<int, std::vector<int>> result;
295  if (pfileIds.empty()) return result;
296 
297  std::string table = queryUploadTreeTableName(uploadId);
298  if (!isValidIdentifier(table)) return result;
299 
300  // Integer-only array; no user input embedded.
301  std::string arr;
302  for (size_t i = 0; i < pfileIds.size(); ++i)
303  {
304  if (i > 0) arr += ",";
305  arr += std::to_string(pfileIds[i]);
306  }
307 
308  bool needsUploadFilter =
309  (table == "uploadtree" || table == "uploadtree_a");
310 
311  QueryResult qr =
312  needsUploadFilter
314  "SELECT uploadtree_pk, pfile_fk FROM %s"
315  " WHERE upload_fk=%d AND pfile_fk=ANY('{%s}'::int[])",
316  table.c_str(), uploadId, arr.c_str())
318  "SELECT uploadtree_pk, pfile_fk FROM %s"
319  " WHERE pfile_fk=ANY('{%s}'::int[])",
320  table.c_str(), arr.c_str());
321 
322  for (int i = 0; i < qr.getRowCount(); ++i)
323  {
324  auto row = qr.getRow(i);
325  int pk = std::stoi(row[0]);
326  int pfileId = std::stoi(row[1]);
327  if (pk > 0 && pfileId > 0)
328  result[pfileId].push_back(pk);
329  }
330  return result;
331 }
332 
333 // Clearing-decision operations
334 
336  int uploadId, int uploadTreeId, int userId, int groupId,
337  int licenseId, bool removed, int type,
338  const std::string& reportInfo, const std::string& comment,
339  const std::string& ack, int jobId)
340 {
341  // Strip Unicode control characters (mirrors PHP StringOperation).
342  std::string safeReport = replaceUnicodeControlChars(reportInfo);
343  std::string safeComment = replaceUnicodeControlChars(comment);
344  std::string safeAck = replaceUnicodeControlChars(ack);
345  const char* removedStr = removed ? "t" : "f";
346 
347  if (jobId <= 0)
348  {
349  // Mark existing decision as WIP first (mirrors ClearingDao::markDecisionAsWip).
350  std::string table = queryUploadTreeTableName(uploadId);
351  if (!isValidIdentifier(table))
352  table = "uploadtree";
353 
355  "INSERT INTO clearing_decision"
356  " (uploadtree_fk, pfile_fk, user_fk, group_fk, decision_type, scope)"
357  " VALUES (%d,"
358  " (SELECT pfile_fk FROM %s WHERE uploadtree_pk=%d),"
359  " %d, %d, 0, 0)",
360  uploadTreeId, table.c_str(), uploadTreeId, userId, groupId);
361 
363  fo_dbManager_PrepareStamement(dbManager.getStruct_dbManager(),
364  "reuserInsertClearingEvent",
365  "INSERT INTO clearing_event"
366  " (uploadtree_fk, user_fk, group_fk, type_fk, rf_fk,"
367  " removed, reportinfo, comment, acknowledgement)"
368  " VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9)"
369  " RETURNING clearing_event_pk",
370  int, int, int, int, int, char*, char*, char*, char*),
371  uploadTreeId, userId, groupId, type, licenseId,
372  removedStr, safeReport.c_str(), safeComment.c_str(), safeAck.c_str());
373 
374  if (!qr || qr.getRowCount() == 0) return 0;
375  return std::stoi(qr.getRow(0)[0]);
376  }
377  else
378  {
380  fo_dbManager_PrepareStamement(dbManager.getStruct_dbManager(),
381  "reuserInsertClearingEventWithJob",
382  "INSERT INTO clearing_event"
383  " (uploadtree_fk, user_fk, group_fk, type_fk, rf_fk,"
384  " removed, reportinfo, comment, acknowledgement, job_fk)"
385  " VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10)"
386  " RETURNING clearing_event_pk",
387  int, int, int, int, int, char*, char*, char*, char*, int),
388  uploadTreeId, userId, groupId, type, licenseId,
389  removedStr, safeReport.c_str(), safeComment.c_str(), safeAck.c_str(),
390  jobId);
391 
392  if (!qr || qr.getRowCount() == 0) return 0;
393  return std::stoi(qr.getRow(0)[0]);
394  }
395 }
396 
398  int uploadId, int uploadTreeId, int userId, int groupId,
399  int decType, int scope, const std::vector<int>& eventIds)
400 {
401  if (eventIds.empty()) return 0;
402 
403  if (!begin()) return 0;
404 
405  // Remove stale WIP decisions for this item/group.
407  fo_dbManager_PrepareStamement(dbManager.getStruct_dbManager(),
408  "reuserRemoveWipDecision",
409  "DELETE FROM clearing_decision"
410  " WHERE uploadtree_fk=$1 AND group_fk=$2 AND decision_type=0",
411  int, int),
412  uploadTreeId, groupId);
413 
414  if (!rRem) { rollback(); return 0; }
415 
416  std::string table = queryUploadTreeTableName(uploadId);
417  if (!isValidIdentifier(table))
418  table = "uploadtree";
419 
421  "INSERT INTO clearing_decision"
422  " (uploadtree_fk, pfile_fk, user_fk, group_fk, decision_type, scope)"
423  " VALUES (%d,"
424  " (SELECT pfile_fk FROM %s WHERE uploadtree_pk=%d),"
425  " %d, %d, %d, %d)"
426  " RETURNING clearing_decision_pk",
427  uploadTreeId, table.c_str(), uploadTreeId, userId, groupId, decType, scope);
428 
429  if (!rIns || rIns.getRowCount() == 0) { rollback(); return 0; }
430  int decisionPk = std::stoi(rIns.getRow(0)[0]);
431 
432  // Link events to the new decision.
433  // Former PHP's ClearingDao::createDecisionFromEvents did not check individual
434  // insert results in the loop (freeResult without error check), so we match
435  // that behaviour: log a warning on failure but continue and commit.
436  auto* stmtLink = fo_dbManager_PrepareStamement(dbManager.getStruct_dbManager(),
437  "reuserInsertClearingDecisionEvent",
438  "INSERT INTO clearing_decision_event"
439  " (clearing_decision_fk, clearing_event_fk) VALUES($1,$2)",
440  int, int);
441 
442  for (int evPk : eventIds)
443  {
444  QueryResult rLink = dbManager.execPrepared(stmtLink, decisionPk, evPk);
445  if (!rLink)
446  LOG_WARNING("Reuser: failed to link clearing_event %d to"
447  " clearing_decision %d, continuing.", evPk, decisionPk);
448  }
449 
450  if (!commit()) { rollback(); return 0; }
451  return decisionPk;
452 }
453 
455  int uploadId, int newItemUploadTreePk, int userId, int groupId,
456  int originalDecisionPk)
457 {
458  // Fetch decision meta (type and scope).
460  fo_dbManager_PrepareStamement(dbManager.getStruct_dbManager(),
461  "reuserGetDecisionMeta",
462  "SELECT decision_type, scope FROM clearing_decision"
463  " WHERE clearing_decision_pk=$1",
464  int),
465  originalDecisionPk);
466 
467  if (!rMeta || rMeta.getRowCount() == 0) return 0;
468  int decType = std::stoi(rMeta.getRow(0)[0]);
469  int scope = std::stoi(rMeta.getRow(0)[1]);
470 
471  // type_fk and job_fk are not copied; copies always use USER type (1).
473  fo_dbManager_PrepareStamement(dbManager.getStruct_dbManager(),
474  "reuserGetEventsForDecision",
475  "SELECT ce.rf_fk, ce.removed,"
476  " ce.reportinfo, ce.comment, ce.acknowledgement"
477  " FROM clearing_event ce"
478  " INNER JOIN clearing_decision_event cde"
479  " ON cde.clearing_event_fk = ce.clearing_event_pk"
480  " WHERE cde.clearing_decision_fk=$1"
481  " ORDER BY ce.clearing_event_pk ASC",
482  int),
483  originalDecisionPk);
484 
485  if (!rEvents) return 0;
486 
487  int jobId = fo_scheduler_jobId();
488  std::vector<int> newEventIds;
489 
490  for (int i = 0; i < rEvents.getRowCount(); ++i)
491  {
492  auto row = rEvents.getRow(i);
493  int rfFk = std::stoi(row[0]);
494  bool isRemoved = (row[1] == "t" || row[1] == "true");
495  // Always use USER type (1) for copied events - mirrors PHP behavior.
496  int evType = 1;
497  int evPk = insertClearingEvent(uploadId, newItemUploadTreePk,
498  userId, groupId,
499  rfFk, isRemoved, evType,
500  row[2], row[3], row[4], jobId);
501  if (evPk > 0)
502  newEventIds.push_back(evPk);
503  }
504 
505  if (newEventIds.empty()) return 0;
506  return createDecisionFromEvents(uploadId, newItemUploadTreePk, userId, groupId,
507  decType, scope, newEventIds);
508 }
509 
510 // ARS record
511 
512 int ReuserDatabaseHandler::writeArsRecord(int agentId, int uploadId,
513  int arsId, bool success)
514 {
515  return fo_WriteARS(dbManager.getConnection(), arsId, uploadId, agentId,
516  "reuser_ars", nullptr, success ? 1 : 0);
517 }
518 
519 // Reuse operations
520 
522  int uploadId, int reusedUploadId,
523  int groupId, int reusedGroupId, int userId)
524 {
525  auto reusedMap = getClearingDecisionMapByPfile(reusedUploadId, reusedGroupId);
526  if (reusedMap.empty()) return true;
527 
528  auto currentMap = getClearingDecisionMapByPfile(uploadId, groupId);
529 
530  // Collect pfiles present in the reused upload but not yet cleared here.
531  std::vector<int> toImport;
532  for (const auto& kv : reusedMap)
533  if (currentMap.find(kv.first) == currentMap.end())
534  toImport.push_back(kv.first);
535 
536  if (toImport.empty()) return true;
537 
538  constexpr size_t chunkSize = 100;
539  for (size_t i = 0; i < toImport.size(); i += chunkSize)
540  {
541  size_t end = std::min(i + chunkSize, toImport.size());
542  std::vector<int> chunk(toImport.begin() + i, toImport.begin() + end);
543  auto pkMap = getUploadTreePksForPfiles(uploadId, chunk);
544 
545  for (const auto& entry : pkMap)
546  {
547  int originalDecision = reusedMap.at(entry.first);
548  for (int uploadtreePk : entry.second)
549  {
550  int newDecision = createCopyOfClearingDecision(
551  uploadId, uploadtreePk, userId, groupId, originalDecision);
552  if (newDecision > 0)
554  }
555  }
556  }
557  return true;
558 }
559 
561  int uploadId, int reusedUploadId,
562  int groupId, int reusedGroupId, int userId)
563 {
564  auto reusedMap = getClearingDecisionMapByPfile(reusedUploadId, reusedGroupId);
565  if (reusedMap.empty()) return true;
566 
567  auto currentMap = getClearingDecisionMapByPfile(uploadId, groupId);
568 
569  std::vector<int> toImport;
570  for (const auto& kv : reusedMap)
571  if (currentMap.find(kv.first) == currentMap.end())
572  toImport.push_back(kv.first);
573 
574  if (toImport.empty()) return true;
575 
576  std::string tableReused = queryUploadTreeTableName(reusedUploadId);
577  std::string tableTarget = queryUploadTreeTableName(uploadId);
578  if (!isValidIdentifier(tableReused) || !isValidIdentifier(tableTarget))
579  return true;
580 
581  bool reusedNeedsFilter = (tableReused == "uploadtree" || tableReused == "uploadtree_a");
582  bool targetNeedsFilter = (tableTarget == "uploadtree" || tableTarget == "uploadtree_a");
583 
584  std::string reusedFilter = reusedNeedsFilter
585  ? " AND ur.upload_fk=" + std::to_string(reusedUploadId) : "";
586  std::string targetFilter = targetNeedsFilter
587  ? " AND ut.upload_fk=" + std::to_string(uploadId) : "";
588 
589  for (int pfileFk : toImport)
590  {
591  int originalDecision = reusedMap.at(pfileFk);
592 
593  std::string reusedPath = getRepoPathOfPfile(pfileFk);
594  if (reusedPath.empty()) continue;
595 
596  // Find items in target upload with matching filename.
598  "SELECT ut.uploadtree_pk, ut.pfile_fk"
599  " FROM %s ur, %s ut"
600  " WHERE ur.pfile_fk=%d%s"
601  " AND ut.ufile_name=ur.ufile_name%s",
602  tableReused.c_str(), tableTarget.c_str(),
603  pfileFk, reusedFilter.c_str(),
604  targetFilter.c_str());
605 
606  for (int i = 0; i < rr.getRowCount(); ++i)
607  {
608  auto row = rr.getRow(i);
609  int newItemPk = std::stoi(row[0]);
610  int newPfileFk = std::stoi(row[1]);
611  if (newItemPk <= 0 || newPfileFk <= 0) continue;
612 
613  std::string newPath = getRepoPathOfPfile(newPfileFk);
614  if (newPath.empty()) continue;
615 
616  int diffCount = diffLineCount(reusedPath, newPath);
617  if (diffCount < 0) return false; // diff failed
618  if (diffCount < 5)
619  {
620  int newDecision = createCopyOfClearingDecision(
621  uploadId, newItemPk, userId, groupId, originalDecision);
622  if (newDecision > 0)
624  }
625  }
626  }
627  return true;
628 }
629 
631  int uploadId, int groupId, int reusedUploadId, int reusedGroupId)
632 {
634  fo_dbManager_PrepareStamement(dbManager.getStruct_dbManager(),
635  "reuserGetReusedMainLicenses",
636  "SELECT rf_fk FROM upload_clearing_license"
637  " WHERE upload_fk=$1 AND group_fk=$2",
638  int, int),
639  reusedUploadId, reusedGroupId);
640 
641  std::set<int> reusedSet;
642  for (int i = 0; i < r1.getRowCount(); ++i)
643  reusedSet.insert(std::stoi(r1.getRow(i)[0]));
644 
645  if (reusedSet.empty()) return true;
646 
648  fo_dbManager_PrepareStamement(dbManager.getStruct_dbManager(),
649  "reuserGetTargetMainLicenses",
650  "SELECT rf_fk FROM upload_clearing_license"
651  " WHERE upload_fk=$1 AND group_fk=$2",
652  int, int),
653  uploadId, groupId);
654 
655  std::set<int> existingSet;
656  for (int i = 0; i < r2.getRowCount(); ++i)
657  existingSet.insert(std::stoi(r2.getRow(i)[0]));
658 
659  for (int rf : reusedSet)
660  {
661  if (existingSet.count(rf)) continue;
663  fo_dbManager_PrepareStamement(dbManager.getStruct_dbManager(),
664  "reuserInsertMainLicense",
665  "INSERT INTO upload_clearing_license (upload_fk, group_fk, rf_fk)"
666  " VALUES ($1,$2,$3)",
667  int, int, int),
668  uploadId, groupId, rf);
669  if (!rIns) return false;
670  }
671  return true;
672 }
673 
675  int uploadId, int reusedUploadId)
676 {
677  // Check that the reused upload has a report_info row.
679  fo_dbManager_PrepareStamement(dbManager.getStruct_dbManager(),
680  "reuserConfSettingsCheck",
681  "SELECT 1 FROM report_info WHERE upload_fk=$1 LIMIT 1",
682  int),
683  reusedUploadId);
684 
685  if (!rCheck || rCheck.getRowCount() == 0) return true;
686 
687  if (!begin()) return false;
688 
689  // Remove any existing report_info for the target upload.
691  fo_dbManager_PrepareStamement(dbManager.getStruct_dbManager(),
692  "reuserConfSettingsDelete",
693  "DELETE FROM report_info WHERE upload_fk=$1",
694  int),
695  uploadId);
696 
697  if (!rDel) { rollback(); return false; }
698 
699  // Dynamically discover columns (excluding pk and upload_fk).
700  // quote_ident ensures safe embedding in the subsequent INSERT.
702  "SELECT string_agg(quote_ident(column_name), ',')"
703  " FROM information_schema.columns"
704  " WHERE table_schema = current_schema()"
705  " AND table_name = 'report_info'"
706  " AND column_name != 'ri_pk'"
707  " AND column_name != 'upload_fk'");
708 
709  if (!rCols || rCols.getRowCount() == 0) { rollback(); return false; }
710  std::string cols = rCols.getRow(0)[0];
711  if (cols.empty()) { rollback(); return false; }
712 
713  // INSERT … SELECT copies all remaining columns from the reused upload.
715  "INSERT INTO report_info(upload_fk, %s)"
716  " SELECT %d, %s FROM report_info WHERE upload_fk=%d",
717  cols.c_str(), uploadId, cols.c_str(), reusedUploadId);
718 
719  if (!rCopy) { rollback(); return false; }
720 
721  if (!commit()) { rollback(); return false; }
722  return true;
723 }
724 
726  int uploadId, int reusedUploadId, int userId)
727 {
728  const std::string agentName = "copyright";
729 
730  // Resolve copyright agent id for both uploads.
732  fo_dbManager_PrepareStamement(dbManager.getStruct_dbManager(),
733  "reuserCopyrightTargetAgentId",
734  "SELECT agent_pk AS agent_id FROM agent"
735  " LEFT JOIN copyright_ars ON agent_fk=agent_pk"
736  " WHERE agent_name=$2 AND agent_enabled"
737  " AND upload_fk=$1 AND ars_success"
738  " ORDER BY agent_pk DESC LIMIT 1",
739  int, char*),
740  uploadId, agentName.c_str());
741 
742  if (!rAgentT || rAgentT.getRowCount() == 0) return true;
743  int targetAgentId = std::stoi(rAgentT.getRow(0)[0]);
744 
746  fo_dbManager_PrepareStamement(dbManager.getStruct_dbManager(),
747  "reuserCopyrightReusedAgentId",
748  "SELECT agent_pk AS agent_id FROM agent"
749  " LEFT JOIN copyright_ars ON agent_fk=agent_pk"
750  " WHERE agent_name=$2 AND agent_enabled"
751  " AND upload_fk=$1 AND ars_success"
752  " ORDER BY agent_pk DESC LIMIT 1",
753  int, char*),
754  reusedUploadId, agentName.c_str());
755 
756  if (!rAgentR || rAgentR.getRowCount() == 0) return true;
757  int reusedAgentId = std::stoi(rAgentR.getRow(0)[0]);
758 
759  // Fetch existing copyright entries in the target upload, keyed by hash.
760  std::string table = queryUploadTreeTableName(uploadId);
761  if (!isValidIdentifier(table)) return true;
762 
763  bool needsUploadFilter = (table == "uploadtree" || table == "uploadtree_a");
764  std::string uploadFilter = needsUploadFilter
765  ? " AND UT.upload_fk = " + std::to_string(uploadId) : "";
766 
768  "SELECT DISTINCT ON (C.copyright_pk, UT.uploadtree_pk)"
769  " C.copyright_pk, UT.uploadtree_pk, UT.upload_fk,"
770  " (CASE WHEN (CE.content IS NULL OR CE.content = '')"
771  " THEN C.content ELSE CE.content END) AS content,"
772  " (CASE WHEN (CE.hash IS NULL OR CE.hash = '')"
773  " THEN C.hash ELSE CE.hash END) AS hash"
774  " FROM copyright C"
775  " INNER JOIN %s UT ON C.pfile_fk = UT.pfile_fk%s"
776  " LEFT JOIN copyright_event CE"
777  " ON CE.copyright_fk = C.copyright_pk"
778  " AND CE.upload_fk = %d"
779  " AND CE.uploadtree_fk = UT.uploadtree_pk"
780  " WHERE C.content IS NOT NULL AND C.content <> ''"
781  " AND (CE.is_enabled IS NULL OR CE.is_enabled = 'true')"
782  " AND C.agent_fk = %d"
783  " ORDER BY C.copyright_pk, UT.uploadtree_pk, content DESC",
784  table.c_str(), uploadFilter.c_str(), uploadId, targetAgentId);
785 
786  // Index existing copyrights by hash.
787  // hash -> list of {copyright_pk, uploadtree_pk, upload_fk}
788  using Row3 = std::array<int, 3>;
789  std::map<std::string, std::vector<Row3>> allMap;
790  for (int i = 0; i < rAll.getRowCount(); ++i)
791  {
792  auto row = rAll.getRow(i);
793  std::string hash = row[4];
794  if (!hash.empty())
795  allMap[hash].push_back(Row3{std::stoi(row[0]), std::stoi(row[1]),
796  std::stoi(row[2])});
797  }
798  if (allMap.empty()) return true;
799 
800  // Fetch copyright events that were modified in the reused upload.
801  // scope 1 == REPO (global events only).
803  fo_dbManager_PrepareStamement(dbManager.getStruct_dbManager(),
804  "reuserGetReusedCopyrightEvents",
805  "SELECT C.copyright_pk, CE.is_enabled, C.hash,"
806  " CE.content AS contentedited"
807  " FROM copyright_event CE"
808  " INNER JOIN copyright C ON C.copyright_pk = CE.copyright_fk"
809  " WHERE CE.upload_fk=$1 AND CE.scope=$3 AND C.agent_fk=$2",
810  int, int, int),
811  reusedUploadId, reusedAgentId, 1 /* REPO scope */);
812 
813  for (int i = 0; i < rReused.getRowCount(); ++i)
814  {
815  auto rRow = rReused.getRow(i);
816  std::string hash = rRow[2];
817  if (hash.empty()) continue;
818 
819  auto it = allMap.find(hash);
820  if (it == allMap.end() || it->second.empty()) continue;
821 
822  Row3 entry = it->second.back();
823  it->second.pop_back();
824  int copyrightPk = entry[0];
825  int uploadtreePk = entry[1];
826  int uploadFk = entry[2];
827  bool isEnabled = fo::stringToBool(rRow[1].c_str());
828  const std::string& contentEdited = rRow[3];
829 
830  // Check if a copyright_event already exists for this combination.
832  fo_dbManager_PrepareStamement(dbManager.getStruct_dbManager(),
833  "reuserCopyrightEventExists",
834  "SELECT EXISTS("
835  " SELECT 1 FROM copyright_event"
836  " WHERE copyright_fk=$1 AND upload_fk=$2 AND uploadtree_fk=$3"
837  ")::int",
838  int, int, int),
839  copyrightPk, uploadFk, uploadtreePk);
840 
841  bool eventExists = rExists && rExists.getRowCount() > 0
842  && std::stoi(rExists.getRow(0)[0]) != 0;
843 
844  // Former PHP's CopyrightDao::updateTable() called getSingleRow() without
845  // checking the return value, and reuseCopyrights() always returned true
846  // regardless. We match that behaviour: log a warning on failure but continue.
847  if (!isEnabled)
848  {
849  QueryResult rWrite =
850  eventExists
852  fo_dbManager_PrepareStamement(dbManager.getStruct_dbManager(),
853  "reuserCopyrightEventDisableUpdate",
854  "UPDATE copyright_event SET scope=$4, is_enabled=false"
855  " WHERE upload_fk=$1 AND copyright_fk=$2 AND uploadtree_fk=$3",
856  int, int, int, int),
857  uploadFk, copyrightPk, uploadtreePk, 1)
859  fo_dbManager_PrepareStamement(dbManager.getStruct_dbManager(),
860  "reuserCopyrightEventDisableInsert",
861  "INSERT INTO copyright_event"
862  " (upload_fk, copyright_fk, uploadtree_fk, is_enabled, scope)"
863  " VALUES($1,$2,$3,'f',$4)",
864  int, int, int, int),
865  uploadFk, copyrightPk, uploadtreePk, 1);
866  if (!rWrite)
867  LOG_WARNING("Reuser: failed to disable copyright_event"
868  " (copyright_fk=%d, uploadtree_fk=%d), continuing.",
869  copyrightPk, uploadtreePk);
870  }
871  else
872  {
873  QueryResult rWrite =
874  eventExists
876  fo_dbManager_PrepareStamement(dbManager.getStruct_dbManager(),
877  "reuserCopyrightEventUpdateContent",
878  "UPDATE copyright_event SET upload_fk=$1, content=$4,"
879  " hash=md5($4)"
880  " WHERE copyright_fk=$2 AND uploadtree_fk=$3",
881  int, int, int, char*),
882  uploadFk, copyrightPk, uploadtreePk, contentEdited.c_str())
884  fo_dbManager_PrepareStamement(dbManager.getStruct_dbManager(),
885  "reuserCopyrightEventInsertContent",
886  "INSERT INTO copyright_event"
887  " (upload_fk, uploadtree_fk, copyright_fk,"
888  " is_enabled, content, hash)"
889  " VALUES($1,$3,$2,'true',$4,md5($4))",
890  int, int, int, char*),
891  uploadFk, copyrightPk, uploadtreePk, contentEdited.c_str());
892  if (!rWrite)
893  LOG_WARNING("Reuser: failed to update copyright_event content"
894  " (copyright_fk=%d, uploadtree_fk=%d), continuing.",
895  copyrightPk, uploadtreePk);
896  }
898  }
899  return true;
900 }
901 
902 std::vector<int> ReuserDatabaseHandler::getPreviousBulkIds(int uploadId, int groupId, int userId)
903 {
904  std::vector<int> bulkIds;
905 
907  fo_dbManager_PrepareStamement(dbManager.getStruct_dbManager(),
908  "reuserGetPreviousBulkIds",
909  "SELECT jq_args FROM upload_reuse, jobqueue, job"
910  " WHERE upload_fk=$1 AND group_fk=$2"
911  " AND (reuse_mode & 8) = 8"
912  " AND EXISTS(SELECT * FROM group_user_member gum WHERE gum.group_fk=upload_reuse.group_fk AND gum.user_fk=$3)"
913  " AND jq_type=$4 AND jq_job_fk=job_pk"
914  " AND job_upload_fk=reused_upload_fk AND job_group_fk=reused_group_fk"
915  " ORDER BY jq_pk",
916  int, int, int, char*),
917  uploadId, groupId, userId, (char*)"monkbulk");
918 
919  std::set<int> seenBulkIds;
920  for (int i = 0; i < qr.getRowCount(); ++i)
921  {
922  std::string jqArgs = qr.getRow(i)[0];
923  std::stringstream ss(jqArgs);
924  std::string line;
925  while (std::getline(ss, line, '\n'))
926  {
927  if (!line.empty())
928  {
929  try {
930  int bulkId = std::stoi(line);
931  if (seenBulkIds.insert(bulkId).second)
932  {
933  bulkIds.push_back(bulkId);
934  }
935  } catch (...) {}
936  }
937  }
938  }
939 
940  return bulkIds;
941 }
942 
943 bool ReuserDatabaseHandler::processBulkReuser(int uploadId, int groupId, int userId)
944 {
945  std::vector<int> bulkIds = getPreviousBulkIds(uploadId, groupId, userId);
946  if (bulkIds.empty()) {
947  return true;
948  }
949 
950  int minTime = 4;
951  int maxTime = 60;
952 
953  for (int bulkId : bulkIds) {
954  int jqPk = rerunBulkAndDeciderOnUpload(uploadId, groupId, userId, bulkId);
955  if (jqPk <= 0) {
956  continue;
957  }
958 
960 
961  const char* fossologyTest = std::getenv("FOSSOLOGY_TEST");
962  if (fossologyTest && std::string(fossologyTest) == "1") {
963  continue;
964  }
965 
966  while (isJobQueueRunning(jqPk)) {
968  int timeInSec = getEstimatedTime(jqPk);
969  if (timeInSec > maxTime) {
970  sleep(maxTime);
971  } else if (timeInSec < minTime) {
972  sleep(minTime);
973  } else {
974  sleep(timeInSec);
975  }
976  }
977  }
978 
979  return true;
980 }
981 
982 int ReuserDatabaseHandler::rerunBulkAndDeciderOnUpload(int uploadId, int groupId, int userId, int bulkId)
983 {
984  ItemTreeBounds bounds;
985  if (!getParentItemBounds(uploadId, bounds)) {
986  return 0;
987  }
988  int nTopItem = bounds.uploadtree_pk;
989 
991  fo_dbManager_PrepareStamement(dbManager.getStruct_dbManager(),
992  "reuserGetBulkUploadtree",
993  "SELECT uploadtree_fk FROM license_ref_bulk WHERE lrb_pk=$1",
994  int),
995  bulkId);
996  if (qrBulk.getRowCount() == 0) {
997  LOG_WARNING("Reuser: no license_ref_bulk row found for lrb_pk %d", bulkId);
998  return 0;
999  }
1000  int bulkUploadtreeFk = 0;
1001  try {
1002  bulkUploadtreeFk = std::stoi(qrBulk.getRow(0)[0]);
1003  } catch (...) {
1004  LOG_WARNING("Reuser: invalid uploadtree_fk for lrb_pk %d", bulkId);
1005  return 0;
1006  }
1007 
1008  int pUID = 0;
1009  std::string ufileName;
1010  int ufileMode = 0;
1011 
1013  fo_dbManager_PrepareStamement(dbManager.getStruct_dbManager(),
1014  "reuserGetUploadtreeEntry1",
1015  "SELECT upload_fk, ufile_name, ufile_mode FROM uploadtree WHERE uploadtree_pk=$1",
1016  int),
1017  bulkUploadtreeFk);
1018  if (qrEntry.getRowCount() > 0) {
1019  pUID = std::stoi(qrEntry.getRow(0)[0]);
1020  ufileName = qrEntry.getRow(0)[1];
1021  ufileMode = std::stoi(qrEntry.getRow(0)[2]);
1022  } else {
1023  QueryResult qrEntry2 = dbManager.execPrepared(
1024  fo_dbManager_PrepareStamement(dbManager.getStruct_dbManager(),
1025  "reuserGetUploadtreeEntry2",
1026  "SELECT upload_fk, ufile_name, ufile_mode FROM uploadtree_a WHERE uploadtree_pk=$1",
1027  int),
1028  bulkUploadtreeFk);
1029  if (qrEntry2.getRowCount() > 0) {
1030  pUID = std::stoi(qrEntry2.getRow(0)[0]);
1031  ufileName = qrEntry2.getRow(0)[1];
1032  ufileMode = std::stoi(qrEntry2.getRow(0)[2]);
1033  } else {
1034  LOG_WARNING("Reuser: no uploadtree entry found for lrb_pk %d uploadtree_fk %d",
1035  bulkId, bulkUploadtreeFk);
1036  return 0;
1037  }
1038  }
1039 
1040  ItemTreeBounds pBounds;
1041  if (!getParentItemBounds(pUID, pBounds)) {
1042  LOG_WARNING("Reuser: getParentItemBounds failed for upload %d", pUID);
1043  return 0;
1044  }
1045  int pTopItem = pBounds.uploadtree_pk;
1046 
1047  int topItem = 0;
1048  if (pTopItem == bulkUploadtreeFk) {
1049  topItem = nTopItem;
1050  } else {
1051  // Find the corresponding entry in the new upload
1053  fo_dbManager_PrepareStamement(dbManager.getStruct_dbManager(),
1054  "reuserGetMatchingEntry1",
1055  "SELECT uploadtree_pk FROM uploadtree WHERE upload_fk=$1 AND ufile_name=$2 AND ufile_mode=$3",
1056  int, char*, int),
1057  uploadId, ufileName.c_str(), ufileMode);
1058  if (qrMatch.getRowCount() > 0) {
1059  try {
1060  topItem = std::stoi(qrMatch.getRow(0)[0]);
1061  } catch (...) {
1062  LOG_WARNING("Reuser: invalid matching uploadtree entry for upload %d", uploadId);
1063  }
1064  }
1065  if (topItem == 0) {
1066  QueryResult qrMatch2 = dbManager.execPrepared(
1067  fo_dbManager_PrepareStamement(dbManager.getStruct_dbManager(),
1068  "reuserGetMatchingEntry2",
1069  "SELECT uploadtree_pk FROM uploadtree_a WHERE upload_fk=$1 AND ufile_name=$2 AND ufile_mode=$3",
1070  int, char*, int),
1071  uploadId, ufileName.c_str(), ufileMode);
1072  if (qrMatch2.getRowCount() > 0) {
1073  try {
1074  topItem = std::stoi(qrMatch2.getRow(0)[0]);
1075  } catch (...) {
1076  LOG_WARNING("Reuser: invalid matching uploadtree_a entry for upload %d", uploadId);
1077  }
1078  }
1079  }
1080  }
1081  if (topItem == 0) {
1082  LOG_WARNING("Reuser: no matching uploadtree entry in new upload %d for"
1083  " ufile_name='%s' ufile_mode=%d", uploadId, ufileName.c_str(), ufileMode);
1084  return 0;
1085  }
1086 
1088  fo_dbManager_PrepareStamement(dbManager.getStruct_dbManager(),
1089  "reuserCloneBulk",
1090  "INSERT INTO license_ref_bulk (user_fk,group_fk,rf_text,upload_fk,uploadtree_fk,ignore_irrelevant,bulk_delimiters,scan_findings) "
1091  "SELECT $1 AS user_fk, $2 AS group_fk,rf_text,$3 AS upload_fk, $4 as uploadtree_fk, ignore_irrelevant, bulk_delimiters, scan_findings "
1092  "FROM license_ref_bulk WHERE lrb_pk=$5 RETURNING lrb_pk",
1093  int, int, int, int, int),
1094  userId, groupId, uploadId, topItem, bulkId);
1095  if (qrClone.getRowCount() == 0) {
1096  LOG_WARNING("Reuser: clone license_ref_bulk failed for lrb_pk %d", bulkId);
1097  return 0;
1098  }
1099  int newBulkId = 0;
1100  try {
1101  newBulkId = std::stoi(qrClone.getRow(0)[0]);
1102  } catch (...) {
1103  LOG_WARNING("Reuser: invalid newBulkId from RETURNING for lrb_pk %d", bulkId);
1104  return 0;
1105  }
1106 
1108  fo_dbManager_PrepareStamement(dbManager.getStruct_dbManager(),
1109  "reuserCloneBulkLic",
1110  "INSERT INTO license_set_bulk (lrb_fk, rf_fk, removing, comment, reportinfo, acknowledgement) "
1111  "SELECT $1 as lrb_fk, rf_fk, removing, comment, reportinfo, acknowledgement FROM license_set_bulk WHERE lrb_fk=$2",
1112  int, int),
1113  newBulkId, bulkId);
1114 
1115  QueryResult qrUpload = dbManager.execPrepared(
1116  fo_dbManager_PrepareStamement(dbManager.getStruct_dbManager(),
1117  "reuserGetUploadFilename",
1118  "SELECT upload_filename FROM upload WHERE upload_pk=$1",
1119  int),
1120  uploadId);
1121  if (qrUpload.getRowCount() == 0) {
1122  LOG_WARNING("Reuser: no upload row found for upload_pk %d", uploadId);
1123  return 0;
1124  }
1125  std::string uploadName = qrUpload.getRow(0)[0];
1126 
1128  fo_dbManager_PrepareStamement(dbManager.getStruct_dbManager(),
1129  "reuserCreateJob",
1130  "INSERT INTO job (job_user_fk, job_group_fk, job_queued, job_priority, job_name, job_upload_fk)"
1131  " VALUES ($1, $2, now(), 0, $3, $4) RETURNING job_pk",
1132  int, int, char*, int),
1133  userId, groupId, uploadName.c_str(), uploadId);
1134  if (qrJob.getRowCount() == 0) {
1135  LOG_WARNING("Reuser: failed to create job for upload %d", uploadId);
1136  return 0;
1137  }
1138  int jobPk = 0;
1139  try {
1140  jobPk = std::stoi(qrJob.getRow(0)[0]);
1141  } catch (...) {
1142  LOG_WARNING("Reuser: invalid job_pk from RETURNING for upload %d", uploadId);
1143  return 0;
1144  }
1145 
1146  std::string newBulkIdStr = std::to_string(newBulkId);
1148  fo_dbManager_PrepareStamement(dbManager.getStruct_dbManager(),
1149  "reuserScheduleMonkBulk",
1150  "INSERT INTO jobqueue (jq_job_fk, jq_type, jq_args, jq_runonpfile, jq_starttime, jq_endtime, jq_end_bits, jq_host, jq_cmd_args)"
1151  " VALUES ($1, 'monkbulk', $2, NULL, NULL, NULL, 0, NULL, NULL) RETURNING jq_pk",
1152  int, char*),
1153  jobPk, newBulkIdStr.c_str());
1154  if (qrMonk.getRowCount() <= 0) {
1155  LOG_WARNING("Reuser: failed to schedule monkbulk for job %d", jobPk);
1156  return 0;
1157  }
1158  int monkJqPk = 0;
1159  try {
1160  monkJqPk = std::stoi(qrMonk.getRow(0)[0]);
1161  } catch (...) {
1162  LOG_WARNING("Reuser: invalid monkbulk jq_pk for job %d", jobPk);
1163  return 0;
1164  }
1165 
1166  QueryResult qrAdjCheck = dbManager.execPrepared(
1167  fo_dbManager_PrepareStamement(dbManager.getStruct_dbManager(),
1168  "reuserCheckAdj2Nest",
1169  "SELECT jq_pk FROM jobqueue, job WHERE job_pk=jq_job_fk"
1170  " AND jq_type='adj2nest' AND job_upload_fk=$1",
1171  int),
1172  uploadId);
1173  int adj2nestJqPk = 0;
1174  if (qrAdjCheck.getRowCount() > 0) {
1175  adj2nestJqPk = std::stoi(qrAdjCheck.getRow(0)[0]);
1176  } else {
1177  std::string uploadIdStr = std::to_string(uploadId);
1179  fo_dbManager_PrepareStamement(dbManager.getStruct_dbManager(),
1180  "reuserScheduleAdj2Nest",
1181  "INSERT INTO jobqueue (jq_job_fk, jq_type, jq_args, jq_runonpfile, jq_starttime, jq_endtime, jq_end_bits, jq_host, jq_cmd_args)"
1182  " VALUES ($1, 'adj2nest', $2, NULL, NULL, NULL, 0, NULL, NULL) RETURNING jq_pk",
1183  int, char*),
1184  jobPk, uploadIdStr.c_str());
1185  if (qrAdj.getRowCount() > 0) {
1186  adj2nestJqPk = std::stoi(qrAdj.getRow(0)[0]);
1187  }
1188  }
1189 
1190  std::string uploadIdStr = std::to_string(uploadId);
1191 
1192  if (!dbManager.begin()) {
1193  LOG_WARNING("Reuser: failed to begin transaction for scheduling upload %d", uploadId);
1194  return 0;
1195  }
1196 
1197  QueryResult qrDecider = dbManager.execPrepared(
1198  fo_dbManager_PrepareStamement(dbManager.getStruct_dbManager(),
1199  "reuserScheduleDecider",
1200  "INSERT INTO jobqueue (jq_job_fk, jq_type, jq_args, jq_runonpfile, jq_starttime, jq_endtime, jq_end_bits, jq_host, jq_cmd_args)"
1201  " VALUES ($1, 'deciderjob', $2, NULL, NULL, NULL, 0, NULL, NULL) RETURNING jq_pk",
1202  int, char*),
1203  jobPk, uploadIdStr.c_str());
1204  if (qrDecider.getRowCount() <= 0) {
1205  LOG_WARNING("Reuser: failed to schedule deciderjob for job %d", jobPk);
1206  dbManager.rollback();
1207  return 0;
1208  }
1209  int deciderJqPk = 0;
1210  try {
1211  deciderJqPk = std::stoi(qrDecider.getRow(0)[0]);
1212  } catch (...) {
1213  LOG_WARNING("Reuser: invalid deciderjob jq_pk for job %d", jobPk);
1214  dbManager.rollback();
1215  return 0;
1216  }
1217 
1218  QueryResult qrDepMonk = dbManager.execPrepared(
1219  fo_dbManager_PrepareStamement(dbManager.getStruct_dbManager(),
1220  "reuserAddDependencyMonk",
1221  "INSERT INTO jobdepends (jdep_jq_fk, jdep_jq_depends_fk) VALUES ($1, $2)",
1222  int, int),
1223  deciderJqPk, monkJqPk);
1224  if (!qrDepMonk) {
1225  LOG_WARNING("Reuser: failed to add monkbulk dependency for job %d", jobPk);
1226  dbManager.rollback();
1227  return 0;
1228  }
1229 
1230  if (adj2nestJqPk > 0) {
1231  QueryResult qrDepAdj = dbManager.execPrepared(
1232  fo_dbManager_PrepareStamement(dbManager.getStruct_dbManager(),
1233  "reuserAddDependencyAdj",
1234  "INSERT INTO jobdepends (jdep_jq_fk, jdep_jq_depends_fk) VALUES ($1, $2)",
1235  int, int),
1236  deciderJqPk, adj2nestJqPk);
1237  if (!qrDepAdj) {
1238  LOG_WARNING("Reuser: failed to add adj2nest dependency for job %d", jobPk);
1239  dbManager.rollback();
1240  return 0;
1241  }
1242  }
1243 
1244  if (!dbManager.commit()) {
1245  LOG_WARNING("Reuser: failed to commit scheduling for job %d", jobPk);
1246  dbManager.rollback();
1247  return 0;
1248  }
1249 
1251  return deciderJqPk;
1252 }
1253 
1255 {
1257  fo_dbManager_PrepareStamement(dbManager.getStruct_dbManager(),
1258  "reuserIsJobQueueRunning",
1259  "SELECT jq_end_bits FROM jobqueue WHERE jq_pk = $1",
1260  int),
1261  jqPk);
1262 
1263  if (qr.getRowCount() == 0) {
1264  return false;
1265  }
1266 
1267  try {
1268  int endBits = std::stoi(qr.getRow(0)[0]);
1269  return !(endBits == 1 || endBits == 2);
1270  } catch (...) {
1271  return false;
1272  }
1273 }
1274 
1276 {
1278  fo_dbManager_PrepareStamement(dbManager.getStruct_dbManager(),
1279  "reuserGetJobFkFromJqPk",
1280  "SELECT jq_job_fk FROM jobqueue WHERE jq_pk = $1",
1281  int),
1282  jqPk);
1283  if (qrJob.getRowCount() == 0) {
1284  return 0;
1285  }
1286  int jobPk = 0;
1287  std::string valJob;
1288  try {
1289  valJob = qrJob.getRow(0)[0];
1290  if (!valJob.empty()) {
1291  jobPk = std::stoi(valJob);
1292  }
1293  } catch (const std::exception& e) {
1294  LOG_WARNING("Reuser: failed to parse job FK for jqPk %d (val='%s'): %s",
1295  jqPk, valJob.c_str(), e.what());
1296  return 0;
1297  } catch (...) {
1298  LOG_WARNING("Reuser: unknown error parsing job FK for jqPk %d (val='%s')",
1299  jqPk, valJob.c_str());
1300  return 0;
1301  }
1302 
1303  QueryResult qrUnpack = dbManager.execPrepared(
1304  fo_dbManager_PrepareStamement(dbManager.getStruct_dbManager(),
1305  "reuserGetUnunpackProcessed",
1306  "SELECT jq_itemsprocessed FROM jobqueue WHERE jq_type = 'ununpack' AND jq_end_bits = 1 AND jq_job_fk = $1",
1307  int),
1308  jobPk);
1309  if (qrUnpack.getRowCount() == 0) {
1310  return 0;
1311  }
1312  int ununpackProcessed = 0;
1313  std::string valUnpack;
1314  try {
1315  valUnpack = qrUnpack.getRow(0)[0];
1316  if (!valUnpack.empty()) {
1317  ununpackProcessed = std::stoi(valUnpack);
1318  }
1319  } catch (const std::exception& e) {
1320  LOG_WARNING("Reuser: failed to parse ununpack processed count (val='%s'): %s",
1321  valUnpack.c_str(), e.what());
1322  return 0;
1323  } catch (...) {
1324  LOG_WARNING("Reuser: unknown error parsing ununpack processed count (val='%s')",
1325  valUnpack.c_str());
1326  return 0;
1327  }
1328 
1329  if (ununpackProcessed <= 0) {
1330  return 0;
1331  }
1332 
1333  QueryResult qrAgents = dbManager.execPrepared(
1334  fo_dbManager_PrepareStamement(dbManager.getStruct_dbManager(),
1335  "reuserGetEstimatedTimeAgents",
1336  "SELECT jq_itemsprocessed, EXTRACT(EPOCH FROM (now() - jq_starttime)) AS burn_time FROM jobqueue "
1337  "WHERE jq_type <> 'ununpack' AND jq_type <> 'reportgen' AND jq_type <> 'decider' AND jq_type <> 'softwareHeritage' "
1338  " AND jq_job_fk = $1 AND jq_endtime IS NULL AND jq_starttime IS NOT NULL",
1339  int),
1340  jobPk);
1341 
1342  double maxCompletionTime = 0.0;
1343  bool foundEstimate = false;
1344 
1345  for (int i = 0; i < qrAgents.getRowCount(); ++i) {
1346  int jqItemsProcessed = 0;
1347  double burnTime = 0.0;
1348  std::string val0, val1;
1349  try {
1350  val0 = qrAgents.getRow(i)[0];
1351  val1 = qrAgents.getRow(i)[1];
1352  if (!val0.empty()) {
1353  jqItemsProcessed = std::stoi(val0);
1354  }
1355  if (!val1.empty()) {
1356  burnTime = std::stod(val1);
1357  }
1358  } catch (const std::exception& e) {
1359  LOG_WARNING("Reuser: failed to parse agent jobqueue row (val0='%s', val1='%s'): %s",
1360  val0.c_str(), val1.c_str(), e.what());
1361  continue;
1362  } catch (...) {
1363  LOG_WARNING("Reuser: unknown error parsing agent jobqueue row (val0='%s', val1='%s')",
1364  val0.c_str(), val1.c_str());
1365  continue;
1366  }
1367 
1368  if (burnTime > 0.0) {
1369  double filesPerSec = static_cast<double>(jqItemsProcessed) / burnTime;
1370  if (filesPerSec > 0.0) {
1371  double timeOfCompletion = static_cast<double>(ununpackProcessed - jqItemsProcessed) / filesPerSec;
1372  if (timeOfCompletion > maxCompletionTime) {
1373  maxCompletionTime = timeOfCompletion;
1374  }
1375  foundEstimate = true;
1376  }
1377  }
1378  }
1379 
1380  if (!foundEstimate) {
1381  return 0;
1382  }
1383 
1384  return static_cast<int>(maxCompletionTime + 0.5);
1385 }
1386 
1388 {
1389  char* host = fo_sysconfig("FOSSOLOGY", "address");
1390  char* port = fo_sysconfig("FOSSOLOGY", "port");
1391  if (!host || !port) return;
1392 
1393  struct addrinfo hints, *servs, *curr = nullptr;
1394  memset(&hints, 0, sizeof(hints));
1395  hints.ai_family = AF_UNSPEC;
1396  hints.ai_socktype = SOCK_STREAM;
1397  if (getaddrinfo(host, port, &hints, &servs) != 0) {
1398  return;
1399  }
1400 
1401  int fd = -1;
1402  for (curr = servs; curr != nullptr; curr = curr->ai_next) {
1403  fd = socket(curr->ai_family, hints.ai_socktype, curr->ai_protocol);
1404  if (fd < 0) continue;
1405  if (connect(fd, curr->ai_addr, curr->ai_addrlen) == 0) {
1406  break;
1407  }
1408  close(fd);
1409  fd = -1;
1410  }
1411  freeaddrinfo(servs);
1412 
1413  if (fd >= 0) {
1414  if (write(fd, "database", 8) < 0) {
1415  // ignore
1416  }
1417  close(fd);
1418  }
1419 }
Database handler for the reuser agent.
virtual bool processUploadReuse(int uploadId, int reusedUploadId, int groupId, int reusedGroupId, int userId)
virtual void notifySchedulerOfDatabaseChange()
virtual bool getParentItemBounds(int uploadId, ItemTreeBounds &out)
Fetch the parent item bounds for a given upload.
virtual int createCopyOfClearingDecision(int uploadId, int newItemUploadTreePk, int userId, int groupId, int originalDecisionPk)
Copy an existing clearing decision to a new uploadtree item.
virtual int createDecisionFromEvents(int uploadId, int uploadTreeId, int userId, int groupId, int decType, int scope, const std::vector< int > &eventIds)
Create a clearing_decision linked to eventIds.
virtual bool reuseMainLicense(int uploadId, int groupId, int reusedUploadId, int reusedGroupId)
virtual bool processBulkReuser(int uploadId, int groupId, int userId)
virtual int getEstimatedTime(int jqPk)
virtual std::vector< int > getPreviousBulkIds(int uploadId, int groupId, int userId)
virtual ReuserDatabaseHandler spawn() const
virtual std::map< int, int > getClearingDecisionMapByPfile(int uploadId, int groupId)
Build a pfile_fk to clearing_decision_pk map for uploadId.
static int getDecisionTypePriority(int decisionType)
Priority for decision types during reuse conflict resolution.
virtual int rerunBulkAndDeciderOnUpload(int uploadId, int groupId, int userId, int bulkId)
virtual bool isJobQueueRunning(int jqPk)
virtual bool processEnhancedUploadReuse(int uploadId, int reusedUploadId, int groupId, int reusedGroupId, int userId)
static bool isValidIdentifier(const std::string &s)
Validate that s contains only characters safe for SQL identifiers.
std::string getRepoPathOfPfile(int pfileId)
virtual int insertClearingEvent(int uploadId, int uploadTreeId, int userId, int groupId, int licenseId, bool removed, int type, const std::string &reportInfo, const std::string &comment, const std::string &ack, int jobId)
Insert a new clearing event and return its primary key (0 on error).
static std::string replaceUnicodeControlChars(const std::string &input)
Strip Unicode control characters (C0, C1, DEL) from input.
virtual bool reuseCopyrights(int uploadId, int reusedUploadId, int userId)
virtual int writeArsRecord(int agentId, int uploadId, int arsId=0, bool success=false)
Write (insert or update) an ARS record.
virtual std::vector< ReuseTriple > getReusedUploads(int uploadId, int groupId)
Return the list of uploads that should be reused for uploadId.
virtual bool reuseConfSettings(int uploadId, int reusedUploadId)
virtual std::map< int, std::vector< int > > getUploadTreePksForPfiles(int uploadId, const std::vector< int > &pfileIds)
For a set of pfile ids, return a map pfile_fk to [uploadtree_pk].
Database handler for agents.
std::string queryUploadTreeTableName(int uploadId)
Get the upload tree table name for a given upload id.
bool commit() const
COMMIT a transaction block in DB.
bool begin() const
BEGIN a transaction block in DB.
char * getPFileNameForFileId(unsigned long pfileId) const
Get the file name of a give pfile id.
DbManager dbManager
DbManager to use.
bool rollback() const
ROLLBACK a transaction block in DB.
DB wrapper for agents.
QueryResult execPrepared(fo_dbManager_PreparedStatement *stmt,...) const
Execute a prepared statement with new parameters.
QueryResult queryPrintf(const char *queryFormat,...) const
Execute a query in printf format.
PGconn * getConnection() const
fo_dbManager * getStruct_dbManager() const
DbManager spawn() const
Wrapper for DB result.
std::vector< std::string > getRow(int i) const
int s
The socket that the CLI will use to communicate.
Definition: fo_cli.c:37
FUNCTION int min(int user_perm, int permExternal)
Get the minimum permission level required.
Definition: libfossagent.c:306
FUNCTION int fo_WriteARS(PGconn *pgConn, int ars_pk, int upload_pk, int agent_pk, const char *tableName, const char *ars_status, int ars_success)
Write ars record.
Definition: libfossagent.c:214
char * fo_RepMkPath(const char *Type, char *Filename)
Given a filename, construct the full path to the file.
Definition: libfossrepo.c:352
void fo_scheduler_heart(int i)
This function must be called by agents to let the scheduler know they are alive and how many items th...
char * fo_sysconfig(const char *sectionname, const char *variablename)
gets a system configuration variable from the configuration data.
int jobId
The id of the job.
int fo_scheduler_jobId()
Gets the id of the job that the agent is running.
fo_dbManager * dbManager
fo_dbManager object
Definition: process.c:16
fo namespace holds the FOSSology library functions.
bool stringToBool(const char *string)
Definition: libfossUtils.cc:32
Bounds of an item within an uploadtree table.
Definition: ReuserTypes.hpp:14