FOSSology  4.7.1
Open Source License Compliance by Open Source Software
adj2nest.c
Go to the documentation of this file.
1 /*
2  adj2nest: Convert adjacency list to nested sets.
3 
4  SPDX-FileCopyrightText: © 2007-2015 Hewlett-Packard Development Company, L.P.
5  SPDX-FileCopyrightText: © 2015 Siemens AG
6 
7  SPDX-License-Identifier: GPL-2.0-only
8 */
9 
66 #include <stdlib.h>
67 #include <stdio.h>
68 #include <unistd.h>
69 #include <string.h>
70 #include <ctype.h>
71 #include <signal.h>
72 #include <libgen.h>
73 
74 #include "libfossology.h"
75 
76 #define MAXCMD 4096
77 #define myBUFSIZ 2048
78 char SQL[256];
79 
80 #ifdef COMMIT_HASH_S
81 char BuildVersion[]="adj2nest build version: " VERSION_S " r(" COMMIT_HASH_S ").\n";
82 #else
83 char BuildVersion[]="adj2nest build version: NULL.\n";
84 #endif
85 
86 PGconn *pgConn = NULL;
87 
92 struct uploadtree
93 {
94  long UploadtreePk;
95  long Child;
96  long Sibling;
97 };
98 typedef struct uploadtree uploadtree;
99 uploadtree *Tree=NULL;
101 long TreeSize=0;
102 long TreeSet=0;
103 long SetNum=0;
104 int isBigUpload=0;
105 /************************************************************/
106 /************************************************************/
107 /************************************************************/
108 
114 void WalkTree (long Index, long Depth)
115 {
116  long LeftSet;
117  PGresult* pgResult;
118  int i;
119 
120  /* Traverse sibling chain iteratively to avoid stack overflow on directories
121  with a large number of files
122  */
123  for (;;)
124  {
125  if (agent_verbose)
126  {
127  for(i=0; i<Depth; i++) printf(" ");
128  LOG_VERBOSE("%ld\n",Tree[Index].UploadtreePk);
129  }
130 
131  LeftSet = SetNum;
132  SetNum++;
133 
134  if (Tree[Index].Child > -1)
135  {
136  WalkTree(Tree[Index].Child,Depth+1);
137  SetNum++;
138  }
139 
140  snprintf(SQL,sizeof(SQL),"UPDATE %s SET lft='%ld', rgt='%ld' WHERE uploadtree_pk='%ld'",
141  uploadtree_tablename,LeftSet,SetNum,Tree[Index].UploadtreePk);
142  pgResult = PQexec(pgConn, SQL);
143  fo_checkPQcommand(pgConn, pgResult, SQL, __FILE__, __LINE__);
144  PQclear(pgResult);
145  fo_scheduler_heart(1);
146 
147  if (Tree[Index].Sibling < 0) break;
148  SetNum++;
149  Index = Tree[Index].Sibling;
150  }
151 
152 } /* WalkTree() */
153 
160 void SetParent (long Parent, long Child)
161 {
162  long P;
163  static long LastParentId=-1;
164  static long LastParentIndex=-1;
165 
166  /* Insert the child */
167  Tree[TreeSet].UploadtreePk = Child;
168  TreeSet++;
169 
170  if (Parent == 0) /* ignore null parent */
171  {
172  return;
173  }
174 
175  /* Find the index of the parent */
176  if (Parent == LastParentId)
177  {
178  P = LastParentIndex;
179  }
180  else
181  {
182  P=0;
183  while((P<TreeSet) && (Tree[P].UploadtreePk != Parent)) { P++; }
184  if (P < TreeSet)
185  {
186  LastParentId = Parent;
187  LastParentIndex = P;
188  }
189  }
190 
191  if (P >= TreeSet)
192  {
193  /* Parent not found, so create it (right after the child). */
194  Tree[TreeSet].UploadtreePk = Parent;
195  Tree[TreeSet].Child = TreeSet-1;
196  LastParentId = Parent;
197  LastParentIndex = TreeSet;
198  TreeSet++;
199  return;
200  }
201 
202  /* Parent found, so follow the chain and add the child to the
203  end of the chain. */
204  if (Tree[P].Child < 0)
205  {
206  Tree[P].Child = TreeSet-1;
207  }
208  else
209  {
210  /* Already have a child so follow that child's sibling chain */
211  P=Tree[P].Child;
212  while(Tree[P].Sibling > -1) P=Tree[P].Sibling; /* find end of the chain */
213  Tree[P].Sibling = TreeSet-1;
214  }
215 } /* SetParent() */
216 
223 void LoadAdj (long UploadPk)
224 {
225  long i;
226  long Parent,Child;
227  long RootRows, NonRootRows;
228  PGresult* pgNonRootResult;
229  PGresult* pgRootResult;
230  PGresult* pgResult;
231  char LastChar;
232 
234 
235  /* If the last character of the uploadtree_tablename is a digit, run analyze */
236  LastChar = uploadtree_tablename[strlen(uploadtree_tablename)-1];
237  if (LastChar >= '0' && LastChar <= '9')
238  {
239  isBigUpload=1;
240  snprintf(SQL,sizeof(SQL),"ANALYZE %s",uploadtree_tablename);
241  pgResult = PQexec(pgConn, SQL);
242  fo_checkPQcommand(pgConn, pgResult, SQL, __FILE__ ,__LINE__);
243  PQclear(pgResult);
244  }
245 
246  snprintf(SQL,sizeof(SQL),"SELECT uploadtree_pk,parent FROM %s WHERE upload_fk = %ld AND parent IS NOT NULL ORDER BY parent, ufile_mode&(1<<29) DESC, ufile_name",uploadtree_tablename,UploadPk);
247  pgNonRootResult = PQexec(pgConn, SQL);
248  fo_checkPQresult(pgConn, pgNonRootResult, SQL, __FILE__, __LINE__);
249 
250  NonRootRows = PQntuples(pgNonRootResult);
251  TreeSize = NonRootRows;
252  LOG_VERBOSE("# Upload %ld: %ld items\n",UploadPk,TreeSize);
253 
254  snprintf(SQL,sizeof(SQL),"SELECT uploadtree_pk,parent FROM %s WHERE upload_fk = %ld AND parent IS NULL",uploadtree_tablename,UploadPk);
255  pgRootResult = PQexec(pgConn, SQL);
256  fo_checkPQresult(pgConn, pgRootResult, SQL, __FILE__, __LINE__);
257 
258  RootRows = PQntuples(pgRootResult);
259  TreeSize += RootRows;
260 
261  /* Got data! Populate the tree! */
262  if (Tree) { free(Tree); }
263  if (TreeSize <= 0) { Tree=NULL; return; }
264  Tree = (uploadtree *)calloc(TreeSize+1,sizeof(uploadtree));
265  for(i=0; i<TreeSize+1; i++)
266  {
267  Tree[i].UploadtreePk=-1;
268  Tree[i].Child=-1;
269  Tree[i].Sibling=-1;
270  }
271 
272  TreeSet=0;
273  SetNum=1;
274 
275  /* Load the roots */
276  for(i=0; i<RootRows; i++)
277  {
278  Child = atol(PQgetvalue(pgRootResult, i, 0));
279  Tree[TreeSet].UploadtreePk = Child;
280  TreeSet++;
281 
282  /* dummy heart to make sure the scheduler knows we are still alive */
283  if ((i % 100000) == 0) fo_scheduler_heart(0);
284  }
285 
286  /* Load all non-roots */
287  for(i=0; i<NonRootRows; i++)
288  {
289  Child = atol(PQgetvalue(pgNonRootResult,i,0));
290  Parent = atol(PQgetvalue(pgNonRootResult,i,1));
291  SetParent(Parent,Child);
292 
293  /* dummy heart to make sure the scheduler knows we are still alive */
294  if ((i % 100000) == 0) fo_scheduler_heart(0);
295  }
296 
297  /* Free up DB memory */
298  PQclear(pgNonRootResult);
299  PQclear(pgRootResult);
300  return;
301 } /* LoadAdj() */
302 
303 /*********************************************
304  Run on all uploads WHERE the upload
305  has no nested set numbers.
306  This displays each upload as it runs!
307  *********************************************/
308 void RunAllNew ()
309 {
310  int Row,MaxRow;
311  long UploadPk;
312  PGresult *pgResult;
313 
314  snprintf(SQL,sizeof(SQL), "SELECT DISTINCT upload_pk,upload_desc,upload_filename FROM upload WHERE upload_pk IN ( SELECT DISTINCT upload_fk FROM uploadtree WHERE lft IS NULL )");
315  pgResult = PQexec(pgConn, SQL);
316  fo_checkPQresult(pgConn, pgResult, SQL, __FILE__, __LINE__);
317 
318  MaxRow = PQntuples(pgResult);
319  for(Row=0; Row < MaxRow; Row++)
320  {
321  UploadPk = atol(PQgetvalue(pgResult,Row,0));
322  if (UploadPk >= 0)
323  {
324  char *S;
325  printf("Processing %ld :: %s",UploadPk,PQgetvalue(pgResult,Row,2));
326  S = PQgetvalue(pgResult,Row,1);
327  if (S && S[0]) printf(" (%s)",S);
328  printf("\n");
329  LoadAdj(UploadPk);
330  if (Tree) WalkTree(0,0);
331  if (Tree) free(Tree);
332  Tree=NULL;
333  TreeSize=0;
334  }
335  }
336  PQclear(pgResult);
337 } /* RunAllNew() */
338 
339 /*********************************************
340  ListUploads(): List every upload ID.
341  *********************************************/
342 void ListUploads ()
343 {
344  int Row,MaxRow;
345  long NewPid;
346  PGresult *pgResult;
347 
348  printf("# Uploads\n");
349  snprintf(SQL,sizeof(SQL), "SELECT upload_pk,upload_desc,upload_filename FROM upload ORDER BY upload_pk");
350  pgResult = PQexec(pgConn, SQL);
351  fo_checkPQresult(pgConn, pgResult, SQL, __FILE__, __LINE__);
352 
353  /* list each value */
354  MaxRow = PQntuples(pgResult);
355  for(Row=0; Row < MaxRow; Row++)
356  {
357  NewPid = atol(PQgetvalue(pgResult,Row,0));
358  if (NewPid >= 0)
359  {
360  char *S;
361  printf("%ld :: %s",NewPid,PQgetvalue(pgResult,Row,2));
362  S = PQgetvalue(pgResult,Row,1);
363  if (S && S[0]) printf(" (%s)",S);
364  printf("\n");
365  }
366  }
367  PQclear(pgResult);
368 } /* ListUploads() */
369 
370 
371 /************************************************************/
372 /************************************************************/
373 /************************************************************/
374 
375 /**********************************************
376  Given a string that contains
377  field='value' pairs, check if the field name
378  matches.
379  \param Field Haystack
380  \param S Needle
381  \return 1 on match, 0 on miss, -1 on no data.
382  **********************************************/
383 int MatchField (char *Field, char *S)
384 {
385  int Len;
386  if (!S || (S[0]=='\0')) return(-1);
387  while(isspace(S[0])) S++;
388  Len = strlen(Field);
389  if (!strncmp(Field,S,Len))
390  {
391  /* Matched string, now make sure it is a real match */
392  while (isspace(S[Len])) Len++;
393  if (S[Len] == '=') return (1);
394  }
395  return(0);
396 } /* MatchField() */
397 
398 /**********************************************
399  Given a string that contains
400  field='value' pairs, skip the first pair and
401  return the pointer to the next pair (or NULL if
402  end of string).
403  \param S field='value' pairs
404  \return Pointer to next pair
405  **********************************************/
406 char * SkipFieldValue (char *S)
407 {
408  char Quote;
409 
410  if (!S || (S[0]=='\0')) return(NULL);
411 
412  /* Skip the field */
413  while((S[0] != '\0') && (S[0]!='=')) S++; /* skip until the '=' is found */
414  if (S[0]=='\0') return(NULL);
415  S++; /* skip the '=' */
416  while(isspace(S[0])) S++; /* Skip any spaces */
417  if (S[0]=='\0') return(NULL);
418 
419  /* Now for the fun part... Skip the Value. This may be quoted. */
420  switch(S[0])
421  {
422  case '\"': case '\'':
423  Quote=S[0];
424  S++;
425  break;
426  default:
427  Quote=' ';
428  break;
429  }
430  while((S[0]!='\0') && (S[0]!=Quote))
431  {
432  if (S[0] == '\\') {
433  S += 2;
434  }
435  else S++;
436  }
437  if (S[0]==Quote) S++;
438  while(isspace(S[0])) S++; /* Skip any spaces */
439  return(S);
440 } /* SkipFieldValue() */
441 
442 /**********************************************
443  The scheduler taints field=value
444  pairs. Given a pair, return the untainted value.
445  NOTE: In string and out string CAN be the same string!
446  NOTE: strlen(Sout) is ALWAYS < strlen(Sin).
447  \param[in] Sin Tainted string
448  \param[out] Sout Untainted string
449  \return Untainted string or NULL if there is an error.
450  **********************************************/
451 char * UntaintValue (char *Sin, char *Sout)
452 {
453  char Quote;
454 
455  /* Skip the field */
456  while((Sin[0] != '\0') && (Sin[0]!='=')) Sin++; /* skip until the '=' is found */
457  if (Sin[0]=='\0') return(NULL);
458  Sin++; /* skip the '=' */
459  while(isspace(Sin[0])) Sin++; /* Skip any spaces */
460  if (Sin[0]=='\0') { Sout[0]='\0'; return(NULL); }
461 
462  /* The value may be inside quotes */
463  switch(Sin[0])
464  {
465  case '\"': case '\'':
466  Quote=Sin[0];
467  Sin++;
468  break;
469  default:
470  Quote=' ';
471  break;
472  }
473 
474  /* Now we're ready to untaint the value */
475  while((Sin[0]!='\0') && (Sin[0]!=Quote))
476  {
477  if (Sin[0] == '\\') {
478  Sin++; /* skip quote char */
479  if (Sin[0] == 'n') {
480  Sout[0] = '\n';
481  }
482  else if (Sin[0] == 'r') {
483  Sout[0] = '\r';
484  }
485  else if (Sin[0] == 'a') {
486  Sout[0] = '\a';
487  }
488  else {
489  Sout[0] = Sin[0];
490  }
491  Sout++;
492  Sin++; /* skip processed char */
493  }
494  else {
495  Sout[0] = Sin[0];
496  Sin++;
497  Sout++;
498  };
499  }
500  Sout[0]='\0'; /* terminate string */
501  return(Sout);
502 } /* UntaintValue() */
503 
504 /**********************************************
505  Convert field=value pairs into parameter.
506  This overwrites the parameter string!
507  The parameter is untainted from the scheduler.
508  \param ParmName field='value' pairs.
509  \param Parm Parameter to be set.
510  \returns 1 if Parm is set, 0 if not.
511  **********************************************/
512 int SetParm (char *ParmName, char *Parm)
513 {
514  int rc;
515  char *OldParm;
516  OldParm=Parm;
517  if (!ParmName || (ParmName[0]=='\0')) return(1); /* no change */
518  if (!Parm || (Parm[0]=='\0')) return(1); /* no change */
519 
520  /* Find the parameter */
521  while(!(rc=MatchField(ParmName,Parm)))
522  {
523  Parm = SkipFieldValue(Parm);
524  }
525  if (rc != 1) return(0); /* no match */
526 
527  /* Found it! Set the value */
528  UntaintValue(Parm,OldParm);
529  return(1);
530 } /* SetParm() */
531 
532 
540 int UpdateUpload(long UploadPk)
541 {
542  PGresult *pgResult;
543 
544  /* update upload.upload_mode to say that adj2nest was successful */
545  snprintf(SQL, sizeof(SQL), "UPDATE upload SET upload_mode = upload_mode | (1<<6) WHERE upload_pk='%ld'",
546  UploadPk);
547  pgResult = PQexec(pgConn, SQL); /* UPDATE upload */
548  if (fo_checkPQcommand(pgConn, pgResult, SQL, __FILE__ ,__LINE__)) return -1;
549  PQclear(pgResult);
550 
551  if(isBigUpload)
552  {
553  snprintf(SQL,sizeof(SQL),"VACUUM ANALYZE %s",uploadtree_tablename);
554  pgResult = PQexec(pgConn, SQL);
555  if (fo_checkPQcommand(pgConn, pgResult, SQL, __FILE__ ,__LINE__)) return -1;
556  PQclear(pgResult);
557  }
558  return(0);
559 }
560 
561 /*********************************************************
562  Usage of the agent
563  \param Name absolute path of the agent
564  *********************************************************/
565 void Usage (char *Name)
566 {
567  printf("Usage: %s [options] [id [id ...]]\n",Name);
568  printf(" -h :: help (print this message), then exit.\n");
569  printf(" -i :: initialize the database, then exit.\n");
570  printf(" -a :: run on ALL uploads that have no nested set records.\n");
571  printf(" -c SYSCONFDIR :: Specify the directory for the system configuration.\n");
572  printf(" -v :: verbose (-vv = more verbose).\n");
573  printf(" -u :: list all upload ids, then exit.\n");
574  printf(" no file :: process upload ids from the scheduler.\n");
575  printf(" id :: process upload ids from the command-line.\n");
576  printf(" -V :: print the version info, then exit.\n");
577 } /* Usage() */
578 
579 /*********************************************************/
580 int main (int argc, char *argv[])
581 {
582  int c, i, rv;
583  long UploadPk=-1;
584  long *uploads_to_scan;
585  int upload_count = 0;
586  int user_pk;
587  char *agent_desc = "Adj2nest Agent";
588  char *COMMIT_HASH;
589  char *VERSION;
590  char agent_rev[myBUFSIZ];
591 
592  /* connect to scheduler. Noop if not run from scheduler. */
593  fo_scheduler_connect(&argc, argv, &pgConn);
594 
595  COMMIT_HASH = fo_sysconfig("adj2nest", "COMMIT_HASH");
596  VERSION = fo_sysconfig("adj2nest", "VERSION");
597  sprintf(agent_rev, "%s.%s", VERSION, COMMIT_HASH);
598  /* Get the Agent Key from the DB */
599  fo_GetAgentKey(pgConn, basename(argv[0]), 0, agent_rev, agent_desc);
600 
601  /* for list of upload_pk's from the command line */
602  uploads_to_scan = calloc(argc, sizeof(long));
603 
604  /* Process command-line */
605  while((c = getopt(argc,argv,"aciuvVh")) != -1)
606  {
607  switch(c)
608  {
609  case 'a': /* run on ALL */
610  RunAllNew();
611  break;
612  case 'c':
613  break; /* handled by fo_scheduler_connect() */
614  case 'i':
615  PQfinish(pgConn);
616  return(0);
617  case 'v': agent_verbose++; break;
618  case 'u':
619  /* list ids */
620  ListUploads();
621  PQfinish(pgConn);
622  return(0);
623  case 'V':
624  printf("%s", BuildVersion);
625  PQfinish(pgConn);
626  return(0);
627  default:
628  Usage(argv[0]);
629  fflush(stdout);
630  PQfinish(pgConn);
631  exit(-1);
632  }
633  }
634 
635  /* Copy filename args (if any) into array */
636  for (i = optind; i < argc; i++)
637  {
638  uploads_to_scan[upload_count] = atol(argv[i]);
639  upload_count++;
640  }
641 
642  if (upload_count == 0)
643  {
644  user_pk = fo_scheduler_userID(); /* get user_pk for user who queued the agent */
645  while(fo_scheduler_next())
646  {
647  UploadPk = atol(fo_scheduler_current());
648 
649  /* Check Permissions */
650  if (GetUploadPerm(pgConn, UploadPk, user_pk) < PERM_WRITE)
651  {
652  LOG_ERROR("You have no update permissions on upload %ld", UploadPk);
653  continue;
654  }
655 
656  LoadAdj(UploadPk);
657  if (Tree) WalkTree(0,0);
658  if (Tree) free(Tree);
659  Tree=NULL;
660  TreeSize=0;
661  /* Update Upload */
662  rv = UpdateUpload(UploadPk);
663  if (rv == -1) LOG_ERROR("Unable to update mode on upload %ld", UploadPk);
664  } /* while() */
665  }
666  else
667  {
668  for (i = 0; i < upload_count; i++)
669  {
670  UploadPk = uploads_to_scan[i];
671  LoadAdj(UploadPk);
672  if (Tree) WalkTree(0,0);
673  if (Tree) free(Tree);
674  Tree=NULL;
675  TreeSize=0;
676  /* Update Upload */
677  rv = UpdateUpload(UploadPk);
678  if (rv == -1) LOG_ERROR("Unable to update mode on upload %ld", UploadPk);
679  }
680  free(uploads_to_scan);
681  }
682 
683  PQfinish(pgConn);
685  return 0;
686 } /* main() */
687 
void LoadAdj(long UploadPk)
Definition: adj2nest.c:223
void SetParent(long Parent, long Child)
Definition: adj2nest.c:160
long SetNum
Definition: adj2nest.c:103
int UpdateUpload(long UploadPk)
Finish updating the upload record and permissions data.
Definition: adj2nest.c:540
char SQL[256]
SQL query to execute.
Definition: adj2nest.c:78
PGconn * pgConn
Database connection.
Definition: adj2nest.c:86
long TreeSet
Definition: adj2nest.c:102
void WalkTree(long Index, long Depth)
Definition: adj2nest.c:114
char * uploadtree_tablename
upload.uploadtree_tablename
Definition: adj2nest.c:100
char BuildVersion[]
Definition: buckets.c:68
Usage()
Print Usage statement.
Definition: fo_dbcheck.php:81
FUNCTION char * GetUploadtreeTableName(PGconn *pgConn, int upload_pk)
Get the uploadtree table name for this upload_pk If upload_pk does not exist, return "uploadtree".
Definition: libfossagent.c:414
FUNCTION int GetUploadPerm(PGconn *pgConn, long UploadPk, int user_pk)
Get users permission to this upload.
Definition: libfossagent.c:378
FUNCTION int fo_GetAgentKey(PGconn *pgConn, const char *agent_name, long Upload_pk, const char *rev, const char *agent_desc)
Get the latest enabled agent key (agent_pk) from the database.
Definition: libfossagent.c:158
int fo_checkPQresult(PGconn *pgConn, PGresult *result, char *sql, char *FileID, int LineNumb)
Check the result status of a postgres SELECT.
Definition: libfossdb.c:170
int fo_checkPQcommand(PGconn *pgConn, PGresult *result, char *sql, char *FileID, int LineNumb)
Check the result status of a postgres commands (not select) If an error occured, write the error to s...
Definition: libfossdb.c:204
The main FOSSology C library.
#define PERM_WRITE
Read-Write permission.
Definition: libfossology.h:33
void fo_scheduler_disconnect(int retcode)
Disconnect the scheduler connection.
char * fo_sysconfig(const char *sectionname, const char *variablename)
gets a system configuration variable from the configuration data.
int agent_verbose
Common verbose flags for the agents, this is used so that the scheduler can change the verbose level ...
int fo_scheduler_userID()
Gets the id of the user that created the job that the agent is running.
char * fo_scheduler_current()
Get the last read string from the scheduler.
char * fo_scheduler_next()
Get the next data to process from the scheduler.
void fo_scheduler_connect(int *argc, char **argv, PGconn **db_conn)
Establish a connection between an agent and the scheduler.
Contains information required by uploadtree elements.
Definition: adj2nest.c:93
long UploadtreePk
Definition: adj2nest.c:94
long Sibling
Definition: adj2nest.c:96
long Child
Definition: adj2nest.c:95