FOSSology  4.5.1
Open Source License Compliance by Open Source Software
utils.c
Go to the documentation of this file.
1 /*
2  SPDX-FileCopyrightText: © 2011-2013 Hewlett-Packard Development Company, L.P.
3  SPDX-FileContributor: Kaushlendra Pratap <kaushlendra-pratap.singh@siemens.com>
4 
5  SPDX-License-Identifier: GPL-2.0-only
6 */
12 #include "ununpack.h"
13 #include "externs.h"
14 #include "regex.h"
15 
19 enum BITS {
20  BITS_PROJECT = 27,
21  BITS_ARTIFACT = 28,
22  BITS_CONTAINER = 29
23 };
24 
28 const char* SCM_REGEX = "/\\.git|\\.hg|\\.bzr|CVS/ROOT|\\.svn/";
29 
30 
31 
41 int IsInflatedFile(char *FileName, int InflateSize)
42 {
43  int result = 0;
44  char FileNameParent[PATH_MAX];
45  struct stat st, stParent;
46  memcpy(FileNameParent, FileName, sizeof(FileNameParent));
47  FileNameParent[PATH_MAX-1] = 0;
48  char *lastSlashPos = strrchr(FileNameParent, '/');
49  if (NULL != lastSlashPos)
50  {
51  /* get the parent container,
52  e.g. for the file ./10g.tar.bz.dir/10g.tar, partent file is ./10g.tar.bz.dir
53  */
54  FileNameParent[lastSlashPos - FileNameParent] = '\0';
55  if (!strcmp(FileNameParent + strlen(FileNameParent) - 4, ".dir"))
56  {
57  /* get the parent file, must be one file
58  e.g. for the file ./10g.tar.bz.dir/10g.tar, partent file is ./10g.tar.bz
59  */
60  FileNameParent[strlen(FileNameParent) - 4] = '\0';
61  stat(FileNameParent, &stParent);
62  stat(FileName, &st);
63  if(S_ISREG(stParent.st_mode) && (st.st_size/stParent.st_size > InflateSize))
64  {
65  result = 1;
66  }
67  }
68  }
69  return result;
70 }
71 
72 
78 void SafeExit (int rc)
79 {
80  if (pgConn) PQfinish(pgConn);
82  exit(rc);
83 } /* SafeExit() */
84 
91 void RemovePostfix(char *Name)
92 {
93  if (NULL == Name) return; // exception
94  // keep the part before the last dot
95  char *LastDot = strrchr(Name, '.');
96  if (LastDot == NULL) return;
97  // if the part after the last dot is number, do not get rid of the postfix
98  if ((LastDot[1]>='0')&&(LastDot[1]<='9')) return;
99  if (LastDot) *LastDot = 0;
100 }
101 
109 void InitCmd ()
110 {
111  int i;
112  PGresult *result;
113 
114  /* clear existing indexes */
115  for(i=0; CMD[i].Magic != NULL; i++)
116  {
117  CMD[i].DBindex = -1; /* invalid value */
118  }
119 
120  if (!pgConn) return; /* DB must be open */
121 
122  /* Load them up! */
123  for(i=0; CMD[i].Magic != NULL; i++)
124  {
125  if (CMD[i].Magic[0] == '\0') continue;
126  ReGetCmd:
127  memset(SQL,'\0',MAXSQL);
128  snprintf(SQL,MAXSQL,"SELECT mimetype_pk FROM mimetype WHERE mimetype_name = '%s';",CMD[i].Magic);
129  result = PQexec(pgConn, SQL); /* SELECT */
130  if (fo_checkPQresult(pgConn, result, SQL, __FILE__, __LINE__)) SafeExit(1);
131  else if (PQntuples(result) > 0) /* if there is a value */
132  {
133  CMD[i].DBindex = atol(PQgetvalue(result,0,0));
134  PQclear(result);
135  }
136  else /* No value, so add it */
137  {
138  PQclear(result);
139  memset(SQL,'\0',MAXSQL);
140  snprintf(SQL,MAXSQL,"INSERT INTO mimetype (mimetype_name) VALUES ('%s');",CMD[i].Magic);
141  result = PQexec(pgConn, SQL); /* INSERT INTO mimetype */
142  if (fo_checkPQcommand(pgConn, result, SQL, __FILE__ ,__LINE__)) SafeExit(2);
143  else
144  {
145  PQclear(result);
146  goto ReGetCmd;
147  }
148  }
149  }
150 } /* InitCmd() */
151 
152 
166 int TaintString (char *Dest, int DestLen,
167  char *Src, int ProtectQuotes, char *Replace)
168 {
169  int i,d;
170  char Temp[FILENAME_MAX];
171 
172  memset(Dest,'\0',DestLen);
173  i=0;
174  d=0;
175  while((Src[i] != '\0') && (d < DestLen))
176  {
177  /* save */
178  if (ProtectQuotes && (Src[i]=='\''))
179  {
180  if (d+4 >= DestLen) return(1);
181  strcpy(Dest+d,"'\\''"); /* unquote, raw quote, requote (for shells) */
182  d+=4;
183  i++;
184  }
185  else if (!ProtectQuotes && strchr("\\",Src[i]))
186  {
187  if (d+2 >= DestLen) return(1);
188  Dest[d] = '\\'; d++;
189  Dest[d] = Src[i]; d++;
190  i++;
191  }
192  else if (Replace && (Src[i]=='%') && (Src[i+1]=='s'))
193  {
194  TaintString(Temp,sizeof(Temp),Replace,1,NULL);
195  if (d+strlen(Temp) >= DestLen) return(1);
196  strcpy(Dest+d,Temp);
197  d = strlen(Dest);
198  i += 2;
199  }
200  else
201  {
202  Dest[d] = Src[i];
203  d++;
204  i++;
205  }
206  }
207  return(0);
208 } /* TaintString() */
209 
218 int Prune (char *Fname, struct stat Stat)
219 {
220  if (!Fname || (Fname[0]=='\0')) return(1); /* not a good name */
221  /* check file type */
222  if (S_ISLNK(Stat.st_mode) || S_ISCHR(Stat.st_mode) ||
223  S_ISBLK(Stat.st_mode) || S_ISFIFO(Stat.st_mode) ||
224  S_ISSOCK(Stat.st_mode))
225  {
226  unlink(Fname);
227  return(1);
228  }
229  /* check hard-link count */
230  if (S_ISREG(Stat.st_mode) && (Stat.st_nlink > 1))
231  {
232  unlink(Fname);
233  return(1);
234  }
235  /* check zero-length files */
236  if (S_ISREG(Stat.st_mode) && (Stat.st_size == 0))
237  {
238  unlink(Fname);
239  return(1);
240  }
241  return(0);
242 } /* Prune() */
243 
249 int MkDirs (char *Fname)
250 {
251  char Dir[FILENAME_MAX+1];
252  int i;
253  int rc=0;
254  struct stat Status;
255 
256  memset(Dir,'\0',sizeof(Dir));
257  strcpy(Dir,Fname);
258  for(i=1; Dir[i] != '\0'; i++)
259  {
260  if (Dir[i] == '/')
261  {
262  Dir[i]='\0';
263  /* Only mkdir if it does not exist */
264  if (stat(Dir,&Status) == 0)
265  {
266  if (!S_ISDIR(Status.st_mode))
267  {
268  LOG_FATAL("'%s' is not a directory.",Dir);
269  SafeExit(3);
270  }
271  }
272  else /* else, it does not exist */
273  {
274  rc=mkdir(Dir,0770); /* create this path segment + Setgid */
275  if (rc && (errno == EEXIST)) rc=0;
276  if (rc)
277  {
278  LOG_FATAL("mkdir %s' failed, error: %s",Dir,strerror(errno));
279  SafeExit(4);
280  }
281  chmod(Dir,02770);
282  } /* else */
283  Dir[i]='/';
284  }
285  }
286  rc = mkdir(Dir,0770); /* create whatever is left */
287  if (rc && (errno == EEXIST)) rc=0;
288  if (rc)
289  {
290  LOG_FATAL("mkdir %s' failed, error: %s",Dir,strerror(errno));
291  SafeExit(5);
292  }
293  chmod(Dir,02770);
294  return(rc);
295 } /* MkDirs() */
296 
304 int MkDir (char *Fname)
305 {
306  if (mkdir(Fname,0770))
307  {
308  if (errno == EEXIST) return(0); /* failed because it exists is ok */
309  return(MkDirs(Fname));
310  }
311  chmod(Fname,02770);
312  return(0);
313 } /* MkDir() */
314 
320 int IsDir (char *Fname)
321 {
322  struct stat Stat;
323  int rc;
324  if (!Fname || (Fname[0]=='\0')) return(0); /* not a directory */
325  rc = lstat(Fname,&Stat);
326  if (rc != 0) return(0); /* bad name */
327  return(S_ISDIR(Stat.st_mode));
328 } /* IsDir() */
329 
336 int IsFile (char *Fname, int Link)
337 {
338  struct stat Stat;
339  int rc;
340  if (!Fname || (Fname[0]=='\0')) return(0); /* not a directory */
341  if (Link) rc = stat(Fname,&Stat);
342  else rc = lstat(Fname,&Stat);
343  if (rc != 0) return(0); /* bad name */
344  return(S_ISREG(Stat.st_mode));
345 } /* IsFile() */
346 
347 
357 int ReadLine (FILE *Fin, char *Line, int MaxLine)
358 {
359  int C;
360  int i;
361 
362  if (!Fin) return(-1);
363  if (feof(Fin)) return(-1);
364  memset(Line,'\0',MaxLine);
365  i=0;
366  C=fgetc(Fin);
367  if (C<0) return(-1);
368  while(!feof(Fin) && (C>=0) && (i<MaxLine))
369  {
370  if (C=='\n')
371  {
372  if (i > 0) return(i);
373  /* if it is a blank line, then ignore it. */
374  }
375  else
376  {
377  Line[i]=C;
378  i++;
379  }
380  C=fgetc(Fin);
381  }
382  return(i);
383 } /* ReadLine() */
384 
394 int IsExe (char *Exe, int Quiet)
395 {
396  char *Path;
397  int i,j;
398  char TestCmd[FILENAME_MAX];
399 
400  Path = getenv("PATH");
401  if (!Path) return(0); /* nope! */
402 
403  memset(TestCmd,'\0',sizeof(TestCmd));
404  j=0;
405  for(i=0; (j<FILENAME_MAX-1) && (Path[i] != '\0'); i++)
406  {
407  if (Path[i]==':')
408  {
409  if ((j>0) && (TestCmd[j-1] != '/')) strcat(TestCmd,"/");
410  strcat(TestCmd,Exe);
411  if (IsFile(TestCmd,1)) return(1); /* found it! */
412  /* missed */
413  memset(TestCmd,'\0',sizeof(TestCmd));
414  j=0;
415  }
416  else
417  {
418  TestCmd[j]=Path[i];
419  j++;
420  }
421  }
422 
423  /* check last path element */
424  if (j>0)
425  {
426  if (TestCmd[j-1] != '/') strcat(TestCmd,"/");
427  strcat(TestCmd,Exe);
428  if (IsFile(TestCmd,1)) return(1); /* found it! */
429  }
430  if (!Quiet) LOG_WARNING("%s not found in $PATH",Exe);
431  return(0); /* not in path */
432 } /* IsExe() */
433 
441 int CopyFile (char *Src, char *Dst)
442 {
443  int Fin, Fout;
444  unsigned char * Mmap;
445  int LenIn, LenOut, Wrote;
446  struct stat Stat;
447  int rc=0;
448  char *Slash;
449 
450  if (lstat(Src,&Stat) == -1) return(1);
451  LenIn = Stat.st_size;
452  if (!S_ISREG(Stat.st_mode)) return(1);
453 
454  Fin = open(Src,O_RDONLY);
455  if (Fin == -1)
456  {
457  LOG_FATAL("Unable to open source '%s'",Src);
458  SafeExit(22);
459  }
460 
461  /* Make sure the directory exists for copying */
462  Slash = strrchr(Dst,'/');
463  if (Slash && (Slash != Dst))
464  {
465  Slash[0]='\0';
466  MkDir(Dst);
467  Slash[0]='/';
468  }
469 
470  Fout = open(Dst,O_WRONLY|O_CREAT|O_TRUNC,Stat.st_mode);
471  if (Fout == -1)
472  {
473  LOG_FATAL("Unable to open target '%s'",Dst);
474  close(Fin);
475  SafeExit(23);
476  }
477 
478  /* load the source file */
479  Mmap = mmap(0,LenIn,PROT_READ,MAP_PRIVATE,Fin,0);
480  if (Mmap == NULL)
481  {
482  LOG_FATAL("pfile %s Unable to process file.",Pfile_Pk);
483  LOG_WARNING("pfile %s Mmap failed during copy.",Pfile_Pk);
484  rc=1;
485  goto CopyFileEnd;
486  }
487 
488  /* write file at maximum speed */
489  LenOut=0;
490  Wrote=0;
491  while((LenOut < LenIn) && (Wrote >= 0))
492  {
493  Wrote = write(Fout,Mmap+LenOut,LenIn-LenOut);
494  LenOut += Wrote;
495  }
496 
497  /* clean up */
498  munmap(Mmap,LenIn);
499  CopyFileEnd:
500  close(Fout);
501  close(Fin);
502  return(rc);
503 } /* CopyFile() */
504 
505 
511 {
512  int i;
513  int Pid;
514  int Status;
515 
516  Pid = wait(&Status);
517  if (Pid <= 0) return(-1); /* no pending children, or call failed */
518 
519  /* find the child! */
520  for(i=0; (i<MAXCHILD) && (Queue[i].ChildPid != Pid); i++) ;
521  if (Queue[i].ChildPid != Pid)
522  {
523  /* child not found */
524  return(-1);
525  }
526 
527  /* check if the child had an error */
528  if (!WIFEXITED(Status))
529  {
530  if (!ForceContinue)
531  {
532  LOG_FATAL("Child had unnatural death");
533  SafeExit(6);
534  }
535  Queue[i].ChildCorrupt=1;
536  Status = -1;
537  }
538  else Status = WEXITSTATUS(Status);
539  if (Status != 0)
540  {
541  if (!ForceContinue)
542  {
543  LOG_FATAL("Child had non-zero status: %d",Status);
544  LOG_FATAL("Child was to recurse on %s",Queue[i].ChildRecurse);
545  SafeExit(10);
546  }
547  Queue[i].ChildCorrupt=1;
548  }
549 
550  /* Finish record */
551  Queue[i].ChildStatus = Status;
552  Queue[i].ChildPid = 0;
553  Queue[i].PI.EndTime = time(NULL);
554  return(i);
555 } /* ParentWait() */
556 
557 /***************************************************************************/
558 /***************************************************************************/
559 /*** Command Processing ***/
560 /***************************************************************************/
561 /***************************************************************************/
562 
568 void CheckCommands (int Show)
569 {
570  int i;
571  int rc;
572 
573  /* Check for CMD_PACK and CMD_ARC tools */
574  for(i=0; CMD[i].Cmd != NULL; i++)
575  {
576  if (CMD[i].Cmd[0] == '\0') continue; /* no command to check */
577  switch(CMD[i].Type)
578  {
579  case CMD_PACK:
580  case CMD_RPM:
581  case CMD_DEB:
582  case CMD_ARC:
583  case CMD_AR:
584  case CMD_PARTITION:
585  case CMD_ZSTD:
586  CMD[i].Status = IsExe(CMD[i].Cmd,Quiet);
587  break;
588  default:
589  ; /* do nothing */
590  }
591  }
592 
593  /* Check for CMD_ISO */
594  rc = ( IsExe("isoinfo",Quiet) && IsExe("grep",Quiet) );
595  for(i=0; CMD[i].Cmd != NULL; i++)
596  {
597  if (CMD[i].Type == CMD_ISO) CMD[i].Status = rc;
598  }
599 
600  /* Check for CMD_DISK */
601  rc = ( IsExe("icat",Quiet) && IsExe("fls",Quiet) );
602  for(i=0; CMD[i].Cmd != NULL; i++)
603  {
604  if (CMD[i].Type == CMD_DISK) CMD[i].Status = rc;
605  }
606 } /* CheckCommands() */
607 
622 int RunCommand (char *Cmd, char *CmdPre, char *File, char *CmdPost,
623  char *Out, char *Where)
624 {
625  char Cmd1[FILENAME_MAX * 5];
626  char CWD[FILENAME_MAX];
627  int rc;
628  char TempPre[FILENAME_MAX];
629  char TempFile[FILENAME_MAX];
630  char TempCwd[FILENAME_MAX];
631  char TempPost[FILENAME_MAX];
632 
633  if (!Cmd) return(0); /* nothing to do */
634 
635  if (Verbose)
636  {
637  if (Where && Out)
638  {
639  LOG_DEBUG("Extracting %s: %s > %s",Cmd,File,Out);
640  }
641  else
642  {
643  if (Where)
644  {
645  LOG_DEBUG("Extracting %s in %s: %s\n",Cmd,Where,File);
646  }
647  else
648  {
649  LOG_DEBUG("Testing %s: %s\n",Cmd,File);
650  }
651  }
652  }
653 
654  if (getcwd(CWD,sizeof(CWD)) == NULL)
655  {
656  LOG_FATAL("directory name longer than %d characters",(int)sizeof(CWD));
657  SafeExit(24);
658  }
659  if (Verbose > 1){ LOG_DEBUG("CWD: %s\n",CWD);}
660  if ((Where != NULL) && (Where[0] != '\0'))
661  {
662  if (chdir(Where) != 0)
663  {
664  MkDir(Where);
665  if (chdir(Where) != 0)
666  {
667  LOG_FATAL("Unable to access directory '%s'",Where);
668  SafeExit(25);
669  }
670  }
671  if (Verbose > 1) LOG_DEBUG("CWD: %s",Where);
672  }
673 
674  /* CMD: Cmd CmdPre 'CWD/File' CmdPost */
675  /* CmdPre and CmdPost may contain a "%s" */
676  memset(Cmd1,'\0',sizeof(Cmd1));
677  if (TaintString(TempPre,FILENAME_MAX,CmdPre,0,Out) ||
678  TaintString(TempFile,FILENAME_MAX,File,1,Out) ||
679  TaintString(TempPost,FILENAME_MAX,CmdPost,0,Out))
680  {
681  return(-1);
682  }
683  if (File[0] != '/')
684  {
685  TaintString(TempCwd,FILENAME_MAX,CWD,1,Out);
686  snprintf(Cmd1,sizeof(Cmd1),"%s %s '%s/%s' %s",
687  Cmd,TempPre,TempCwd,TempFile,TempPost);
688  }
689  else
690  {
691  snprintf(Cmd1,sizeof(Cmd1),"%s %s '%s' %s",
692  Cmd,TempPre,TempFile,TempPost);
693  }
694  rc = system(Cmd1);
695  if (WIFSIGNALED(rc))
696  {
697  LOG_ERROR("Process killed by signal (%d): %s",WTERMSIG(rc),Cmd1);
698  SafeExit(8);
699  }
700  if (WIFEXITED(rc)) rc = WEXITSTATUS(rc);
701  else rc=-1;
702  if (Verbose) LOG_DEBUG("in %s -- %s ; rc=%d",Where,Cmd1,rc);
703 
704  if(chdir(CWD) != 0)
705  LOG_ERROR("Unable to change directory to %s", CWD);
706  if (Verbose > 1) LOG_DEBUG("CWD: %s",CWD);
707  return(rc);
708 } /* RunCommand() */
709 
710 
717 {
718  MagicCookie = magic_open(MAGIC_MIME);
719  if (MagicCookie == NULL)
720  {
721  LOG_FATAL("Failed to initialize magic cookie");
722  SafeExit(9);
723  }
724  return magic_load(MagicCookie,NULL);
725 }
726 
735 {
736  /* Set .dsc file magic as application/x-debian-source */
737  char *pExt;
738  FILE *fp;
739  char line[500];
740  int j;
741  char c;
742 
743  pExt = strrchr(Filename, '.');
744  if ( pExt != NULL)
745  {
746  if (strcmp(pExt, ".dsc")==0)
747  {
748  /* read the first 500 characters of the file to verify that
749  * it really is a debian source file
750  */
751  if ((fp = fopen(Filename, "r")) == NULL) return 0;
752  j=0;
753  while ((c = fgetc(fp)) != EOF && j < 500 ){
754  line[j]=c;
755  j++;
756  }
757  fclose(fp);
758  if ((strstr(line, "-----BEGIN PGP SIGNED MESSAGE-----") && strstr(line,"Source:")) ||
759  (strstr(line, "Format:") && strstr(line, "Source:") && strstr(line, "Version:")))
760  {
761  return 1;
762  }
763  }
764  }
765  return 0;
766 }
767 
773 void OctetType(char *Filename, char *TypeBuf)
774 {
775  int rc1, rc2, rc3;
776  char *Type;
777 
778  /* Get more information from magic */
779  magic_setflags(MagicCookie, MAGIC_NONE);
780  Type = (char *)magic_file(MagicCookie, Filename);
781  /* reset magic flags */
782  magic_setflags(MagicCookie, MAGIC_MIME);
783 
784  /* .deb and .udeb as application/x-debian-package*/
785  if (strstr(Type, "Debian binary package"))
786  {
787  strcpy(TypeBuf,"application/x-debian-package");
788  return;
789  }
790 
791  if (strstr(Type, "ISO 9660"))
792  {
793  strcpy(TypeBuf,"application/x-iso9660-image");
794  return;
795  }
796 
797  /* 7zr can handle many formats (including isos), so try this first */
798  rc1 = RunCommand("7z","l -y ",Filename,">/dev/null 2>&1",NULL,NULL);
799  rc2 = RunCommand("7z","t -y -pjunk",Filename,">/dev/null 2>&1",NULL,NULL);
800  if(rc2!=0)
801  {
802  rc3 = RunCommand("7z","t -y -pjunk",Filename,"|grep 'Wrong password' >/dev/null 2>&1",NULL,NULL);
803  if(rc3==0)
804  {
805  LOG_ERROR("'%s' cannot be unpacked, password required.",Filename);
806  return;
807  }
808  }
809  if ((rc1 || rc2)==0)
810  {
811  strcpy(TypeBuf,"application/x-7z-w-compressed");
812  return;
813  }
814 
815  if (strstr(Type, " ext2 "))
816  {
817  strcpy(TypeBuf,"application/x-ext2");
818  return;
819  }
820 
821  if (strstr(Type, " ext3 "))
822  {
823  strcpy(TypeBuf,"application/x-ext3");
824  return;
825  }
826 
827  if (strstr(Type, "x86 boot sector, mkdosfs")) /* the file type is FAT */
828  {
829  strcpy(TypeBuf,"application/x-fat");
830  return;
831  }
832 
833  if (strstr(Type, "NTFS")) /* the file type is NTFS */
834  {
835  strcpy(TypeBuf,"application/x-ntfs");
836  return;
837  }
838 
839  if (strstr(Type, "x86 boot")) /* the file type is boot partition */
840  {
841  strcpy(TypeBuf,"application/x-x86_boot");
842  return;
843  }
844 }
845 
851 int FindCmd (char *Filename)
852 {
853  char *Type;
854  char TypeBuf[256];
855  int Match;
856  int i;
857  int rc;
858 
859  if (!MagicCookie) InitMagic();
860  TypeBuf[0] = 0;
861 
862  Type = (char *)magic_file(MagicCookie,Filename);
863  if (Type == NULL) return(-1);
864 
865  /* Windows executables look like archives and 7z will attempt to unpack them.
866  * If that happens there will be a .bss file representing the .bss segment.
867  * 7z will try to unpack this further, potentially getting into an infinite
868  * unpack loop. So if you see an octet type .bss file, consider it text
869  * to avoid this problem. This problem was first noticed on papi-4.1.3-3.el6.src.rpm
870  */
871  if ((strcmp(basename(Filename), ".bss") == 0) && (strstr(Type, "octet")))
872  {
873  Type = strdup("text/plain");
874  }
875 
876  /* The Type returned by magic_file needs to be verified and possibly rewritten.
877  * So save it in a static buffer.
878  */
879  strncpy(TypeBuf, Type, sizeof(TypeBuf));
880  TypeBuf[255] = 0; /* make sure TypeBuf is null terminated */
881 
882  if (strstr(Type, "octet" ))
883  {
884  OctetType(Filename, TypeBuf);
885  }
886  else
887  if (IsDebianSourceFile(Filename)) strcpy(TypeBuf,"application/x-debian-source");
888  else
889  if (strstr(Type, "msword") || strstr(Type, "vnd.ms"))
890  strcpy(TypeBuf, "application/x-7z-w-compressed");
891  else
892  /* some files you just have to try to verify their type */
893  if (strstr(Type, "application/x-exe") ||
894  strstr(Type, "application/x-shellscript"))
895  {
896  rc = RunCommand("unzip","-q -l",Filename,">/dev/null 2>&1",NULL,NULL);
897  if ((rc==0) || (rc==1) || (rc==2) || (rc==51))
898  {
899  strcpy(TypeBuf,"application/x-zip");
900  }
901  } /* if was x-exe */
902  else if (strstr(Type, "application/x-tar"))
903  {
904  if (RunCommand("tar","-tf",Filename,">/dev/null 2>&1",NULL,NULL) != 0)
905  return(-1); /* bad tar! (Yes, they do happen) */
906  } /* if was x-tar */
907 
908  /* Match Type (mimetype from magic or from special processing above to determine
909  * the command for Filename
910  */
911  Match=-1;
912  for(i=0; (CMD[i].Cmd != NULL) && (Match == -1); i++)
913  {
914  if (CMD[i].Status == 0) continue; /* cannot check */
915  if (CMD[i].Type == CMD_DEFAULT)
916  {
917  Match=i; /* done! */
918  }
919  else
920  if (!strstr(TypeBuf, CMD[i].Magic))
921  {
922  continue; /* not a match */
923  }
924  Match=i;
925  }
926 
927  if (Verbose > 0)
928  {
929  /* no match */
930  if (Match == -1)
931  {
932  LOG_DEBUG("MISS: Type=%s %s",TypeBuf,Filename);
933  }
934  else
935  {
936  LOG_DEBUG("MATCH: Type=%d %s %s %s %s",CMD[Match].Type,CMD[Match].Cmd,CMD[Match].CmdPre,Filename,CMD[Match].CmdPost);
937  }
938  }
939  return(Match);
940 } /* FindCmd() */
941 
942 /***************************************************************************/
943 /***************************************************************************/
944 /*** File Processing ***/
945 /***************************************************************************/
946 /***************************************************************************/
947 
953 {
954  dirlist *d;
955  /* free records */
956  while(DL)
957  {
958  d=DL; /* grab the head */
959  DL=DL->Next; /* increment new head */
960  /* free old head */
961  if (d->Name) free(d->Name);
962  free(d);
963  }
964 } /* FreeDirList() */
965 
971 dirlist * MakeDirList (char *Fullname)
972 {
973  dirlist *dlist=NULL, *dhead=NULL;
974  DIR *Dir;
975  struct dirent *Entry;
976 
977  /* no effort is made to sort since all records need to be processed anyway */
978  /* Current order is "reverse inode order" */
979  Dir = opendir(Fullname);
980  if (Dir == NULL) return(NULL);
981 
982  Entry = readdir(Dir);
983  while(Entry != NULL)
984  {
985  if (!strcmp(Entry->d_name,".")) goto skip;
986  if (!strcmp(Entry->d_name,"..")) goto skip;
987  dhead = (dirlist *)malloc(sizeof(dirlist));
988  if (!dhead)
989  {
990  LOG_FATAL("Failed to allocate dirlist memory");
991  SafeExit(10);
992  }
993  dhead->Name = (char *)malloc(strlen(Entry->d_name)+1);
994  if (!dhead->Name)
995  {
996  LOG_FATAL("Failed to allocate dirlist.Name memory");
997  SafeExit(11);
998  }
999  memset(dhead->Name,'\0',strlen(Entry->d_name)+1);
1000  strcpy(dhead->Name,Entry->d_name);
1001  /* add record to the list */
1002  dhead->Next = dlist;
1003  dlist = dhead;
1004 #if 0
1005  {
1006  /* bubble-sort name -- head is out of sequence */
1008  char *Name;
1009  dhead = dlist;
1010  while(dhead->Next && (strcmp(dhead->Name,dhead->Next->Name) > 0))
1011  {
1012  /* swap names */
1013  Name = dhead->Name;
1014  dhead->Name = dhead->Next->Name;
1015  dhead->Next->Name = Name;
1016  dhead = dhead->Next;
1017  }
1018  }
1019 #endif
1020 
1021  skip:
1022  Entry = readdir(Dir);
1023  }
1024  closedir(Dir);
1025 
1026 #if 0
1027  /* debug: List the directory */
1028  printf("Directory: %s\n",Fullname);
1029  for(dhead=dlist; dhead; dhead=dhead->Next)
1030  {
1031  printf(" %s\n",dhead->Name);
1032  }
1033 #endif
1034 
1035  return(dlist);
1036 } /* MakeDirList() */
1037 
1048 void SetDir (char *Dest, int DestLen, char *Smain, char *Sfile)
1049 {
1050  int i;
1051 
1052  memset(Dest,'\0',DestLen);
1053  if (Smain)
1054  {
1055  strcpy(Dest,Smain);
1056  /* remove absolute path (stay in destination) */
1057  if (Sfile && (Sfile[0]=='/')) Sfile++;
1058  /* skip "../" */
1061  i=1;
1062  while(i && Sfile)
1063  {
1064  i=0;
1065  if (!memcmp(Sfile,"../",3)) { Sfile+=3; i=1; }
1066  else if (!memcmp(Sfile,"./",2)) { Sfile+=2; i=1; }
1067  }
1068  while(Sfile && !memcmp(Sfile,"../",3)) Sfile+=3;
1069  }
1070 
1071  if ((strlen(Dest) > 0) && (Last(Smain) != '/') && (Sfile[0] != '/'))
1072  strcat(Dest,"/");
1073  if (Sfile) strcat(Dest,Sfile);
1074  /* remove terminating file */
1075  for(i=strlen(Dest)-1; (i>=0) && (Dest[i] != '/'); i--)
1076  {
1077  Dest[i]='\0';
1078  }
1079 } /* SetDir() */
1080 
1081 
1087 {
1088  LOG_DEBUG("Container:");
1089  printf(" Source: %s\n",CI->Source);
1090  printf(" Partdir: %s\n",CI->Partdir);
1091  printf(" Partname: %s\n",CI->Partname);
1092  printf(" PartnameNew: %s\n",CI->PartnameNew);
1093  printf(" TopContainer: %d\n",CI->TopContainer);
1094  printf(" HasChild: %d\n",CI->HasChild);
1095  printf(" Pruned: %d\n",CI->Pruned);
1096  printf(" Corrupt: %d\n",CI->Corrupt);
1097  printf(" Artifact: %d\n",CI->Artifact);
1098  printf(" IsDir: %d\n",CI->IsDir);
1099  printf(" IsCompressed: %d\n",CI->IsCompressed);
1100  printf(" uploadtree_pk: %ld\n",CI->uploadtree_pk);
1101  printf(" pfile_pk: %ld\n",CI->pfile_pk);
1102  printf(" ufile_mode: %ld\n",CI->ufile_mode);
1103  printf(" Parent Cmd: %d\n",CI->PI.Cmd);
1104  printf(" Parent ChildRecurseArtifact: %d\n",CI->PI.ChildRecurseArtifact);
1105  printf(" Parent uploadtree_pk: %ld\n",CI->PI.uploadtree_pk);
1106 } /* DebugContainerInfo() */
1107 
1115 int DBInsertPfile (ContainerInfo *CI, char *Fuid)
1116 {
1117  PGresult *result;
1118  char *Val; /* string result from SQL query */
1119  long tempMimeType;
1120  char *tempSha256;
1121 
1122  /* idiot checking */
1123  if (!Fuid || (Fuid[0] == '\0')) return(1);
1124 
1125  /* Check if the pfile exists */
1126  memset(SQL,'\0',MAXSQL);
1127  snprintf(SQL,MAXSQL,"SELECT pfile_pk,pfile_mimetypefk,pfile_sha256 FROM pfile "
1128  "WHERE pfile_sha1 = '%.40s' AND pfile_md5 = '%.32s' AND pfile_size = '%s';",
1129  Fuid,Fuid+41,Fuid+140);
1130  result = PQexec(pgConn, SQL); /* SELECT */
1131  if (fo_checkPQresult(pgConn, result, SQL, __FILE__, __LINE__)) SafeExit(12);
1132 
1133  /* add it if it was not found */
1134  if (PQntuples(result) == 0)
1135  {
1136  /* blindly insert to pfile table in database (don't care about dups) */
1137  /* If TWO ununpacks are running at the same time, they could both
1138  create the same pfile at the same time. Ignore the dup constraint. */
1139  PQclear(result);
1140  memset(SQL,'\0',MAXSQL);
1141  if (CMD[CI->PI.Cmd].DBindex > 0)
1142  {
1143  snprintf(SQL,MAXSQL,"INSERT INTO pfile (pfile_sha1,pfile_md5,pfile_sha256,pfile_size,pfile_mimetypefk) "
1144  "VALUES ('%.40s','%.32s','%.64s','%s','%ld');",
1145  Fuid,Fuid+41,Fuid+74,Fuid+140,CMD[CI->PI.Cmd].DBindex);
1146  }
1147  else
1148  {
1149  snprintf(SQL,MAXSQL,"INSERT INTO pfile (pfile_sha1,pfile_md5,pfile_sha256,pfile_size) VALUES ('%.40s','%.32s','%.64s','%s');",
1150  Fuid,Fuid+41,Fuid+74,Fuid+140);
1151  }
1152  result = PQexec(pgConn, SQL); /* INSERT INTO pfile */
1153  // ignore duplicate constraint failure (23505), report others
1154  if ((result==0) || ((PQresultStatus(result) != PGRES_COMMAND_OK) &&
1155  (strncmp("23505", PQresultErrorField(result, PG_DIAG_SQLSTATE),5))))
1156  {
1157  LOG_ERROR("Error inserting pfile, %s.", SQL);
1158  SafeExit(13);
1159  }
1160  PQclear(result);
1161 
1162  /* Now find the pfile_pk. Since it might be a dup, we cannot rely
1163  on currval(). */
1164  memset(SQL,'\0',MAXSQL);
1165  snprintf(SQL,MAXSQL,"SELECT pfile_pk,pfile_mimetypefk,pfile_sha256 FROM pfile "
1166  "WHERE pfile_sha1 = '%.40s' AND pfile_md5 = '%.32s' AND pfile_sha256 = '%.64s' AND pfile_size = '%s';",
1167  Fuid,Fuid+41,Fuid+74,Fuid+140);
1168  result = PQexec(pgConn, SQL); /* SELECT */
1169  if (fo_checkPQresult(pgConn, result, SQL, __FILE__, __LINE__)) SafeExit(14);
1170  }
1171 
1172  /* Now *DB contains the pfile_pk information */
1173  Val = PQgetvalue(result,0,0);
1174  if (Val)
1175  {
1176  CI->pfile_pk = atol(Val);
1177  if (Verbose) LOG_DEBUG("pfile_pk = %ld",CI->pfile_pk);
1178  tempMimeType = atol(PQgetvalue(result,0,1));
1179  tempSha256 = PQgetvalue(result,0,2);
1180  /* For backwards compatibility... Do we need to update the mimetype? */
1181  if ((CMD[CI->PI.Cmd].DBindex > 0) &&
1182  ((tempMimeType != CMD[CI->PI.Cmd].DBindex)))
1183  {
1184  PQclear(result);
1185  memset(SQL,'\0',MAXSQL);
1186  snprintf(SQL,MAXSQL,"UPDATE pfile SET pfile_mimetypefk = '%ld' WHERE pfile_pk = '%ld';",
1187  CMD[CI->PI.Cmd].DBindex, CI->pfile_pk);
1188  result = PQexec(pgConn, SQL); /* UPDATE pfile */
1189  if (fo_checkPQcommand(pgConn, result, SQL, __FILE__ ,__LINE__)) SafeExit(16);
1190  }
1191  /* Update the SHA256 for the pfile if it does not exists */
1192  if (strncasecmp(tempSha256, Fuid+74, 64) != 0)
1193  {
1194  PQclear(result);
1195  memset(SQL,'\0',MAXSQL);
1196  snprintf(SQL,MAXSQL,"UPDATE pfile SET pfile_sha256 = '%.64s' WHERE pfile_pk = '%ld';",
1197  Fuid+74, CI->pfile_pk);
1198  result = PQexec(pgConn, SQL); /* UPDATE pfile */
1199  if (fo_checkPQcommand(pgConn, result, SQL, __FILE__ ,__LINE__)) SafeExit(16);
1200  }
1201  PQclear(result);
1202  }
1203  else
1204  {
1205  PQclear(result);
1206  CI->pfile_pk = -1;
1207  return(0);
1208  }
1209 
1210  return(1);
1211 } /* DBInsertPfile() */
1212 
1225 int TestSCMData(char *sourcefilename)
1226 {
1227  regex_t preg;
1228  int err;
1229  int found=0;
1230 
1231  err = regcomp (&preg, SCM_REGEX, REG_NOSUB | REG_EXTENDED);
1232  if (err == 0)
1233  {
1234  int match;
1235 
1236  match = regexec (&preg, sourcefilename, 0, NULL, 0);
1237  regfree (&preg);
1238  if(match == 0)
1239  {
1240  found = 1;
1241  if (Verbose) LOG_DEBUG("match found %s",sourcefilename);
1242  }
1243  else if(match == REG_NOMATCH)
1244  {
1245  found = 0;
1246  if (Verbose) LOG_DEBUG("match not found %s",sourcefilename);
1247  }
1248  else
1249  {
1250  char *text;
1251  size_t size;
1252  size = regerror (err, &preg, NULL, 0);
1253  text = malloc (sizeof (*text) * size);
1254  if(text)
1255  {
1256  regerror (err, &preg, text, size);
1257  LOG_ERROR("Error regexc '%s' '%s' return %d, error %s",SCM_REGEX,sourcefilename,match,text);
1258  }
1259  else
1260  {
1261  LOG_ERROR("Not enough memory (%lu)",sizeof (*text) * size);
1262  SafeExit(127);
1263  }
1264  found = 0;
1265  }
1266  }
1267  else
1268  {
1269  LOG_ERROR("Error regcomp(%d)",err);
1270  SafeExit(127);
1271  }
1272 
1273 
1274  return(found);
1275 } /* TestSCMData() */
1276 
1288 {
1289  char UfileName[1024];
1290  char *cp;
1291  PGresult *result;
1292  char EscBuf[1024];
1293  int error;
1294 
1295  if (!Upload_Pk) return(-1); /* should never happen */
1296  // printf("=========== BEFORE ==========\n"); DebugContainerInfo(CI);
1297 
1298  /* Find record's mode */
1299  CI->ufile_mode = CI->Stat.st_mode & Mask;
1300  if (!CI->TopContainer && CI->Artifact) CI->ufile_mode |= (1 << BITS_ARTIFACT);
1301  if (CI->HasChild) CI->ufile_mode |= (1 << BITS_CONTAINER);
1302 
1303  /* Find record's name */
1304  memset(UfileName,'\0',sizeof(UfileName));
1305  if (CI->TopContainer)
1306  {
1307  char *ufile_name;
1308  snprintf(UfileName,sizeof(UfileName),"SELECT upload_filename FROM upload WHERE upload_pk = %s;",Upload_Pk);
1309  result = PQexec(pgConn, UfileName);
1310  if (fo_checkPQresult(pgConn, result, UfileName, __FILE__, __LINE__)) SafeExit(17);
1311  memset(UfileName,'\0',sizeof(UfileName));
1312  ufile_name = PQgetvalue(result,0,0);
1313  PQclear(result);
1314  if (strchr(ufile_name,'/')) ufile_name = strrchr(ufile_name,'/')+1;
1315  strncpy(CI->Partname,ufile_name,sizeof(CI->Partname)-1);
1316  }
1317  else if (CI->Artifact)
1318  {
1319  int Len;
1320  Len = strlen(CI->Partname);
1321  /* determine type of artifact */
1322  if ((Len > 4) && !strcmp(CI->Partname+Len-4,".dir"))
1323  strcpy(UfileName,"artifact.dir");
1324  else if ((Len > 9) && !strcmp(CI->Partname+Len-9,".unpacked"))
1325  strcpy(UfileName,"artifact.unpacked");
1326  else if ((Len > 5) && !strcmp(CI->Partname+Len-5,".meta"))
1327  strcpy(UfileName,"artifact.meta");
1328  else /* Don't know what it is */
1329  strcpy(UfileName,"artifact");
1330  strncpy(CI->Partname,UfileName,sizeof(CI->Partname)-1);
1331  }
1332 
1333  PQescapeStringConn(pgConn, EscBuf, CI->Partname, strlen(CI->Partname), &error);
1334  if (error)
1335  {
1336  LOG_WARNING("Error escaping filename with multibyte character set (%s).", CI->Partname);
1337  }
1338  else
1339  {
1340  strncpy(UfileName, EscBuf, sizeof(UfileName));
1341  }
1342 
1343  /*
1344  * Tests for SCM Data: IgnoreSCMData is global and defined in ununpack_globals.h with false value
1345  * and pass to true if ununpack is called with -I option to ignore SCM data.
1346  * So if IgnoreSCMData is false the right test is true.
1347  * Otherwise if IgnoreSCMData is true and CI->Source is not a SCM data then add it in database.
1348  */
1349  if(ReunpackSwitch && ((IgnoreSCMData && !TestSCMData(CI->Source)) || !IgnoreSCMData))
1350  {
1351  /* postgres 8.3 seems to have a problem escaping binary characters
1352  * (it works in 8.4). So manually substitute '~' for any unprintable and slash chars.
1353  */
1354  for (cp=UfileName; *cp; cp++) if (!isprint(*cp) || (*cp=='/') || (*cp=='\\')) *cp = '~';
1355 
1356  /* Get the parent ID */
1357  /* Two cases -- depending on if the parent exists */
1358  memset(SQL,'\0',MAXSQL);
1359  if (CI->PI.uploadtree_pk > 0) /* This is a child */
1360  {
1361  /* Prepare to insert child */
1362  snprintf(SQL,MAXSQL,"INSERT INTO %s (parent,pfile_fk,ufile_mode,ufile_name,upload_fk) VALUES (%ld,%ld,%ld,E'%s',%s);",
1364  UfileName, Upload_Pk);
1365  result = PQexec(pgConn, SQL); /* INSERT INTO uploadtree */
1366  if (fo_checkPQcommand(pgConn, result, SQL, __FILE__ ,__LINE__))
1367  {
1368  SafeExit(18);
1369  }
1370  PQclear(result);
1371  }
1372  else /* No parent! This is the first upload! */
1373  {
1374  snprintf(SQL,MAXSQL,"INSERT INTO %s (upload_fk,pfile_fk,ufile_mode,ufile_name) VALUES (%s,%ld,%ld,E'%s');",
1375  uploadtree_tablename, Upload_Pk, CI->pfile_pk, CI->ufile_mode, UfileName);
1376  result = PQexec(pgConn, SQL); /* INSERT INTO uploadtree */
1377  if (fo_checkPQcommand(pgConn, result, SQL, __FILE__ ,__LINE__)) SafeExit(19);
1378  PQclear(result);
1379  }
1380  /* Find the inserted child */
1381  memset(SQL,'\0',MAXSQL);
1382  snprintf(SQL,MAXSQL,"SELECT currval('uploadtree_uploadtree_pk_seq');");
1383  result = PQexec(pgConn, SQL);
1384  if (fo_checkPQresult(pgConn, result, SQL, __FILE__, __LINE__)) SafeExit(20);
1385  CI->uploadtree_pk = atol(PQgetvalue(result,0,0));
1386  PQclear(result);
1387  }
1388  TotalItems++;
1389  fo_scheduler_heart(1);
1390  return(0);
1391 } /* DBInsertUploadTree() */
1392 
1403 int AddToRepository (ContainerInfo *CI, char *Fuid, int Mask)
1404 {
1405  int IsUnique = 1; /* is it a DB replica? */
1406 
1407  /*****************************************/
1408  /* populate repository (include artifacts) */
1409  /* If we ever want to skip artifacts, use && !CI->Artifact */
1410  if ((Fuid[0]!='\0') && UseRepository)
1411  {
1412  /* Translate the new Fuid into old Fuid */
1413  char FuidNew[1024];
1414  memset(FuidNew, '\0', sizeof(FuidNew));
1415  // Copy the value till md5
1416  strncpy(FuidNew, Fuid, 74);
1417  // Copy the size of the file
1418  strcat(FuidNew,Fuid+140);
1419 
1420  /* put file in repository */
1421  if (!fo_RepExist(REP_FILES,Fuid))
1422  {
1423  if (fo_RepImport(CI->Source,REP_FILES,FuidNew,1) != 0)
1424  {
1425  LOG_ERROR("Failed to import '%s' as '%s' into the repository",CI->Source,FuidNew);
1426  SafeExit(21);
1427  }
1428  }
1429  if (Verbose) LOG_DEBUG("Repository[%s]: insert '%s' as '%s'",
1430  REP_FILES,CI->Source,FuidNew);
1431  }
1432 
1433  /* PERFORMANCE NOTE:
1434  I used to use and INSERT and an UPDATE.
1435  Turns out, INSERT is fast, UPDATE is *very* slow (10x).
1436  Now I just use an INSERT.
1437  */
1438 
1439  /* Insert pfile record */
1440  if (pgConn && Upload_Pk)
1441  {
1442  if (!DBInsertPfile(CI,Fuid)) return(0);
1443  /* Update uploadtree table */
1444  IsUnique = !DBInsertUploadTree(CI,Mask);
1445  }
1446 
1447  if (ForceDuplicate) IsUnique=1;
1448  return(IsUnique);
1449 } /* AddToRepository() */
1450 
1459 {
1460  int i;
1461  int Mask=0177000; /* used for XML modemask */
1462  char Fuid[1024];
1463 
1464  if (CI->Source[0] == '\0') return(0);
1465  memset(Fuid,0,sizeof(Fuid));
1466  /* TotalItems++; */
1467 
1468  /* list source */
1469  if (ListOutFile)
1470  {
1471  fputs("<item source=\"",ListOutFile);
1472  for(i=0; CI->Source[i] != '\0'; i++)
1473  {
1474  if (isalnum(CI->Source[i]) ||
1475  strchr(" `~!@#$%^*()-=_+[]{}\\|;:',./?",CI->Source[i]))
1476  fputc(CI->Source[i],ListOutFile);
1477  else fprintf(ListOutFile,"&#x%02x;",(int)(CI->Source[i])&0xff);
1478  }
1479  fputs("\" ",ListOutFile);
1480 
1481  /* list file names */
1482  if (CI->Partname[0] != '\0')
1483  {
1484  fputs("name=\"",ListOutFile);
1485  /* XML taint-protect name */
1486  for(i=0; CI->Partname[i] != '\0'; i++)
1487  {
1488  if (isalnum(CI->Partname[i]) ||
1489  strchr(" `~!@#$%^*()-=_+[]{}\\|;:',./?",CI->Partname[i]))
1490  fputc(CI->Partname[i],ListOutFile);
1491  else fprintf(ListOutFile,"&#x%02x;",(int)(CI->Partname[i])&0xff);
1492  }
1493  fputs("\" ",ListOutFile);
1494  }
1495 
1496  /* list mime info */
1497  if ((CI->PI.Cmd >= 0) && (CMD[CI->PI.Cmd].Type != CMD_DEFAULT))
1498  {
1499  fprintf(ListOutFile,"mime=\"%s\" ",CMD[CI->PI.Cmd].Magic);
1500  TotalFiles++;
1501  }
1502  else if (S_ISDIR(CI->Stat.st_mode))
1503  {
1504  fprintf(ListOutFile,"mime=\"directory\" ");
1505  TotalDirectories++;
1506  }
1507  else TotalFiles++;
1508 
1509  /* identify compressed files */
1510  if (CMD[CI->PI.Cmd].Type == CMD_PACK)
1511  {
1512  fprintf(ListOutFile,"compressed=\"1\" ");
1514  }
1515  /* identify known artifacts */
1516  if (CI->Artifact)
1517  {
1518  fprintf(ListOutFile,"artifact=\"1\" ");
1519  TotalArtifacts++;
1520  }
1521 
1522  if (CI->HasChild) fprintf(ListOutFile,"haschild=\"1\" ");
1523  } /* if ListOutFile */
1524 
1525  if (!CI->TopContainer)
1526  {
1527  /* list mode */
1528  Mask=0177000;
1529  if (Cmd >= 0)
1530  {
1531  if (S_ISDIR(CI->Stat.st_mode))
1532  {
1533  Mask = CMD[Cmd].ModeMaskDir;
1534  }
1535  else if (S_ISREG(CI->Stat.st_mode))
1536  {
1537  Mask = CMD[Cmd].ModeMaskReg;
1538  }
1539  }
1540 
1541  if (ListOutFile)
1542  {
1543  if (!CI->Artifact) /* no masks for an artifact */
1544  {
1545  fprintf(ListOutFile,"mode=\"%07o\" ",CI->Stat.st_mode & Mask);
1546  fprintf(ListOutFile,"modemask=\"%07o\" ",Mask);
1547  }
1548 
1549  /* identify known corrupted files */
1550  if (CI->Corrupt) fprintf(ListOutFile,"error=\"%d\" ",CI->Corrupt);
1551 
1552  /* list timestamps */
1553  if (CI->Stat.st_mtime)
1554  {
1555  if ((CI->Stat.st_mtime < CI->PI.StartTime) || (CI->Stat.st_mtime > CI->PI.EndTime))
1556  fprintf(ListOutFile,"mtime=\"%d\" ",(int)(CI->Stat.st_mtime));
1557  }
1558 #if 0
1559  /* commented out since almost anything can screw this up. */
1560  if (CI->Stat.st_ctime)
1561  {
1562  if ((CI->Stat.st_ctime < CI->PI.StartTime) || (CI->Stat.st_ctime > CI->PI.EndTime))
1563  fprintf(ListOutFile,"ctime=\"%d\" ",(int)(CI->Stat.st_ctime));
1564  }
1565 #endif
1566  } /* if ListOutFile */
1567  } /* if not top container */
1568 
1569  /* list checksum info for files only! */
1570  if (S_ISREG(CI->Stat.st_mode) && !CI->Pruned)
1571  {
1572  CksumFile *CF;
1573  Cksum *Sum;
1574  char SHA256[65];
1575 
1576  memset(SHA256, '\0', sizeof(SHA256));
1577 
1578  CF = SumOpenFile(CI->Source);
1579  if(calc_sha256sum(CI->Source, SHA256))
1580  {
1581  LOG_FATAL("Unable to calculate SHA256 of %s\n", CI->Source);
1582  SafeExit(56);
1583  }
1584 
1585  if (CF)
1586  {
1587  Sum = SumComputeBuff(CF);
1588  SumCloseFile(CF);
1589 
1590  if (Sum)
1591  {
1592  for(i=0; i<20; i++) { sprintf(Fuid+0+i*2,"%02X",Sum->SHA1digest[i]); }
1593  Fuid[40]='.';
1594  for(i=0; i<16; i++) { sprintf(Fuid+41+i*2,"%02X",Sum->MD5digest[i]); }
1595  Fuid[73]='.';
1596  for(i=0; i<64; i++) { sprintf(Fuid+74+i,"%c",SHA256[i]); }
1597  Fuid[139]='.';
1598  snprintf(Fuid+140,sizeof(Fuid)-140,"%Lu",(long long unsigned int)Sum->DataLen);
1599  if (ListOutFile) fprintf(ListOutFile,"fuid=\"%s\" ",Fuid);
1600  free(Sum);
1601  } /* if Sum */
1602  } /* if CF */
1603  else /* file too large to mmap (probably) */
1604  {
1605  FILE *Fin;
1606  Fin = fopen(CI->Source,"rb");
1607  if (Fin)
1608  {
1609  Sum = SumComputeFile(Fin);
1610  if (Sum)
1611  {
1612  for(i=0; i<20; i++) { sprintf(Fuid+0+i*2,"%02X",Sum->SHA1digest[i]); }
1613  Fuid[40]='.';
1614  for(i=0; i<16; i++) { sprintf(Fuid+41+i*2,"%02X",Sum->MD5digest[i]); }
1615  Fuid[73]='.';
1616  for(i=0; i<64; i++) { sprintf(Fuid+74+i,"%c",SHA256[i]); }
1617  Fuid[139]='.';
1618  snprintf(Fuid+140,sizeof(Fuid)-140,"%Lu",(long long unsigned int)Sum->DataLen);
1619  if (ListOutFile) fprintf(ListOutFile,"fuid=\"%s\" ",Fuid);
1620  free(Sum);
1621  }
1622  fclose(Fin);
1623  }
1624  }
1625  } /* if is file */
1626 
1627  /* end XML */
1628  if (ListOutFile)
1629  {
1630  if (CI->HasChild) fputs(">\n",ListOutFile);
1631  else fputs("/>\n",ListOutFile);
1632  } /* if ListOutFile */
1633 
1634  return(AddToRepository(CI,Fuid,Mask));
1635 } /* DisplayContainerInfo() */
1636 
1642 int RemoveDir(char *dirpath)
1643 {
1644  char RMcmd[FILENAME_MAX];
1645  int rc;
1646  memset(RMcmd, '\0', sizeof(RMcmd));
1647  snprintf(RMcmd, FILENAME_MAX -1, "rm -rf '%s' ", dirpath);
1648 // nokill = fo_scheduler_get_special(SPECIAL_NOKILL);
1649  rc = system(RMcmd);
1650 // fo_scheduler_set_special(SPECIAL_NOKILL, nokill);
1651  return rc;
1652 } /* RemoveDir() */
1653 
1654 
1663 char *PathCheck(char *DirPath)
1664 {
1665  char *NewPath;
1666  char *subs;
1667  char TmpPath[2048];
1668  char HostName[2048];
1669  struct timeval time_st;
1670 
1671  NewPath = strdup(DirPath);
1672 
1673  if ((subs = strstr(NewPath,"%U")) )
1674  {
1675  /* dir substitution */
1676  if (gettimeofday(&time_st, 0))
1677  {
1678  /* gettimeofday failure */
1679  LOG_WARNING("gettimeofday() failure.");
1680  time_st.tv_usec = 999999;
1681  }
1682 
1683  *subs = 0;
1684  snprintf(TmpPath, sizeof(TmpPath), "%s%ul", NewPath, (unsigned)time_st.tv_usec);
1685  free(NewPath);
1686  NewPath = strdup(TmpPath);
1687  }
1688 
1689  if ((subs = strstr(NewPath,"%H")) )
1690  {
1691  /* hostname substitution */
1692  gethostname(HostName, sizeof(HostName));
1693 
1694  *subs = 0;
1695  snprintf(TmpPath, sizeof(TmpPath), "%s%s%s", NewPath, HostName, subs+2);
1696  free(NewPath);
1697  NewPath = strdup(TmpPath);
1698  }
1699 #ifndef STANDALONE
1700  if ((subs = strstr(NewPath, "%R")) )
1701  {
1702  /* repo location substitution */
1703  *subs = 0;
1704 
1705  snprintf(TmpPath, sizeof(TmpPath), "%s%s%s", NewPath, fo_config_get(sysconfig, "FOSSOLOGY", "path", NULL), subs+2);
1706  free(NewPath);
1707  NewPath = strdup(TmpPath);
1708  }
1709 #endif
1710  return(NewPath);
1711 }
1712 
1713 void deleteTmpFiles(char *dir)
1714 {
1715  if (strcmp(dir, ".")) RemoveDir(dir);
1716 }
1717 
1718 
1724 void Usage (char *Name, char *Version)
1725 {
1726  fprintf(stderr,"Universal Unpacker, %s, compiled %s %s\n",
1727  Version,__DATE__,__TIME__);
1728  fprintf(stderr,"Usage: %s [options] file [file [file...]]\n",Name);
1729  fprintf(stderr," Extracts each file.\n");
1730  fprintf(stderr," If filename specifies a directory, then extracts everything in it.\n");
1731  fprintf(stderr," Unpack Options:\n");
1732  fprintf(stderr," -h :: help (print this message), then exit.\n");
1733  fprintf(stderr," -C :: force continue when unpack tool fails.\n");
1734  fprintf(stderr," -d dir :: specify alternate extraction directory. %%U substitutes a unique ID.\n");
1735  fprintf(stderr," Default is the same directory as file (usually not a good idea).\n");
1736  fprintf(stderr," -m # :: number of CPUs to use (default: 1).\n");
1737  fprintf(stderr," -P :: prune files: remove links, >1 hard links, zero files, etc.\n");
1738  fprintf(stderr," -R :: recursively unpack (same as '-r -1')\n");
1739  fprintf(stderr," -r # :: recurse to a specified depth (0=none/default, -1=infinite)\n");
1740  fprintf(stderr," -X :: remove recursive sources after unpacking.\n");
1741  fprintf(stderr," -x :: remove ALL unpacked files when done (clean up).\n");
1742  fprintf(stderr," I/O Options:\n");
1743  fprintf(stderr," -L out :: Generate a log of files extracted (in XML) to out.\n");
1744  fprintf(stderr," -F :: Using files from the repository.\n");
1745  fprintf(stderr," -i :: Initialize the database queue system, then exit.\n");
1746  fprintf(stderr," -I :: Ignore SCM Data.\n");
1747  fprintf(stderr," -Q :: Using scheduler queue system. (Includes -F)\n");
1748  fprintf(stderr," If -L is used, unpacked files are placed in 'files'.\n");
1749  fprintf(stderr," -T rep :: Set gold repository name to 'rep' (for testing)\n");
1750  fprintf(stderr," -t rep :: Set files repository name to 'rep' (for testing)\n");
1751  fprintf(stderr," -A :: do not set the initial DB container as an artifact.\n");
1752  fprintf(stderr," -f :: force processing files that already exist in the DB.\n");
1753  fprintf(stderr," -q :: quiet (generate no output).\n");
1754  fprintf(stderr," -U upload_pk :: upload to unpack (implies -RQ). Writes to db.\n");
1755  fprintf(stderr," -v :: verbose (-vv = more verbose).\n");
1756  fprintf(stderr," -V :: print the version info, then exit.\n");
1757  fprintf(stderr,"Currently identifies and processes:\n");
1758  fprintf(stderr," Compressed files: .Z .gz .bz .bz2 upx\n");
1759  fprintf(stderr," Archives files: tar cpio zip jar ar rar cab\n");
1760  fprintf(stderr," Data files: pdf\n");
1761  fprintf(stderr," Installer files: rpm deb\n");
1762  fprintf(stderr," File images: iso9660(plain/Joliet/Rock Ridge) FAT(12/16/32) ext2/ext3 NTFS\n");
1763  fprintf(stderr," Boot partitions: x86, vmlinuz\n");
1765 } /* Usage() */
1766 
1773  void SQLNoticeProcessor(void *arg, const char *message)
1774  {
1775  }
1776 
1788  int ShouldExclude(char *Filename, const char *ExcludePatterns)
1789  {
1790  if (!ExcludePatterns || !Filename) return 0;
1791 
1792  char *patternsCopy = strdup(ExcludePatterns);
1793  if (!patternsCopy) return 0;
1794 
1795  char *pattern = strtok(patternsCopy, ",");
1796  while (pattern != NULL) {
1797  if (strstr(Filename, pattern)) {
1798  if (Verbose) LOG_DEBUG("Excluding: %s (matched substring: %s)", Filename, pattern);
1799  free(patternsCopy);
1800  return 1;
1801  }
1802  pattern = strtok(NULL, ",");
1803  }
1804  free(patternsCopy);
1805  return 0;
1806  }
char SQL[256]
SQL query to execute.
Definition: adj2nest.c:78
char * uploadtree_tablename
upload.uploadtree_tablename
Definition: adj2nest.c:100
Cksum * SumComputeBuff(CksumFile *CF)
Compute the checksum, allocate and return a Cksum containing the sum value.
Definition: checksum.c:182
Cksum * SumComputeFile(FILE *Fin)
Compute the checksum, allocate and return a string containing the sum value.
Definition: checksum.c:115
CksumFile * SumOpenFile(char *Fname)
Open and mmap a file.
Definition: checksum.c:26
void SumCloseFile(CksumFile *CF)
Close a file that was opened with SumOpenFile()
Definition: checksum.c:83
int Verbose
Verbose level.
Definition: util.c:19
PGconn * pgConn
Database connection.
Definition: adj2nest.c:86
Stores all extern variables used by the agent.
char * Pfile_Pk
Pfile pk in DB.
int ReunpackSwitch
Set if the uploadtree records are missing from db.
int UseRepository
Using files from the repository?
int TotalDirectories
Number of directories.
unpackqueue Queue[MAXCHILD+1]
Manage children.
int Quiet
Run in quiet mode?
int TotalFiles
Number of regular files.
int TotalArtifacts
Number of artifacts.
cmdlist CMD[]
Global command table.
char * Upload_Pk
Upload pk in DB.
char REP_FILES[16]
Files repository name.
FILE * ListOutFile
File to store unpack list.
long TotalItems
Number of records inserted.
int IgnoreSCMData
1: Ignore SCM data, 0: dont ignore it.
int TotalCompressedFiles
Number of compressed files.
int ForceDuplicate
When using db, should it process duplicates?
int ForceContinue
Force continue when unpack tool fails?
magic_t MagicCookie
for Magic
Definition: finder.c:23
char * fo_config_get(fo_conf *conf, const char *group, const char *key, GError **error)
Gets an element based on its group name and key name. If the group or key is not found,...
Definition: fossconfig.c:336
int fo_checkPQresult(PGconn *pgConn, PGresult *result, char *sql, char *FileID, int LineNumb)
Check the result status of a postgres SELECT.
Definition: libfossdb.c:170
int fo_checkPQcommand(PGconn *pgConn, PGresult *result, char *sql, char *FileID, int LineNumb)
Check the result status of a postgres commands (not select) If an error occured, write the error to s...
Definition: libfossdb.c:204
int fo_RepImport(char *Source, char *Type, char *Filename, int Link)
Import a file into the repository.
Definition: libfossrepo.c:812
int fo_RepExist(char *Type, char *Filename)
Determine if a file exists.
Definition: libfossrepo.c:486
void fo_scheduler_disconnect(int retcode)
Disconnect the scheduler connection.
void fo_scheduler_heart(int i)
This function must be called by agents to let the scheduler know they are alive and how many items th...
fo_conf * sysconfig
Store file handler and mmap of a file.
Definition: checksum.h:44
Store check sum of a file.
Definition: checksum.h:33
uint8_t SHA1digest[20]
SHA1 digest of the file.
Definition: checksum.h:35
uint64_t DataLen
Size of the file.
Definition: checksum.h:36
uint8_t MD5digest[16]
MD5 digest of the file.
Definition: checksum.h:34
Structure for storing information about a particular file.
Definition: ununpack.h:116
int IsCompressed
Definition: ununpack.h:129
long pfile_pk
Definition: ununpack.h:131
int TopContainer
Definition: ununpack.h:121
ParentInfo PI
Definition: ununpack.h:126
char Partname[FILENAME_MAX]
Definition: ununpack.h:119
long uploadtree_pk
Definition: ununpack.h:130
long ufile_mode
Definition: ununpack.h:132
struct stat Stat
Definition: ununpack.h:125
char PartnameNew[FILENAME_MAX]
Definition: ununpack.h:120
char Partdir[FILENAME_MAX]
Definition: ununpack.h:118
Definition: monk.h:61
Definition: match.h:20
time_t StartTime
Definition: ununpack.h:78
long uploadtree_pk
Definition: ununpack.h:81
int ChildRecurseArtifact
Definition: ununpack.h:80
time_t EndTime
Definition: ununpack.h:79
cmdtype Type
Definition: ununpack.h:149
long DBindex
Definition: ununpack.h:153
int Status
Definition: ununpack.h:150
char * Cmd
Definition: ununpack.h:145
int ModeMaskReg
Definition: ununpack.h:152
int ModeMaskDir
Definition: ununpack.h:151
Directory linked list.
Definition: ununpack.h:105
struct dirlist * Next
Definition: ununpack.h:107
Store the results of a regex match.
Definition: scanners.hpp:28
int ChildStatus
Definition: ununpack.h:92
ParentInfo PI
Definition: ununpack.h:97
int ChildCorrupt
Definition: ununpack.h:93
static char * Dst
Destination location.
Definition: test_CopyFile.c:14
static char * Src
Souce location.
Definition: test_CopyFile.c:13
int RunCommand(char *Cmd, char *CmdPre, char *File, char *CmdPost, char *Out, char *Where)
Try a command and return command code.
Definition: utils.c:622
int IsFile(char *Fname, int Link)
Given a filename, is it a file?
Definition: utils.c:336
const char * SCM_REGEX
Definition: utils.c:28
int DisplayContainerInfo(ContainerInfo *CI, int Cmd)
Print what can be printed in XML.
Definition: utils.c:1458
void DebugContainerInfo(ContainerInfo *CI)
Print a ContainerInfo structure.
Definition: utils.c:1086
int RemoveDir(char *dirpath)
Remove all files under dirpath (rm -rf)
Definition: utils.c:1642
int MkDirs(char *Fname)
Same as command-line "mkdir -p".
Definition: utils.c:249
int Prune(char *Fname, struct stat Stat)
Given a filename and its stat, prune it.
Definition: utils.c:218
int TestSCMData(char *sourcefilename)
Search for SCM data in the filename.
Definition: utils.c:1225
int InitMagic()
Open and load Magic file Initializes global MagicCookie.
Definition: utils.c:716
void CheckCommands(int Show)
Make sure all commands are usable.
Definition: utils.c:568
int IsDebianSourceFile(char *Filename)
Read file to see if it is a Debian source file.
Definition: utils.c:734
int MkDir(char *Fname)
Smart mkdir.
Definition: utils.c:304
int TaintString(char *Dest, int DestLen, char *Src, int ProtectQuotes, char *Replace)
Protect strings intelligently.
Definition: utils.c:166
void SetDir(char *Dest, int DestLen, char *Smain, char *Sfile)
Set a destination directory name.
Definition: utils.c:1048
int ParentWait()
Wait for a child. Sets child status.
Definition: utils.c:510
void RemovePostfix(char *Name)
get rid of the postfix
Definition: utils.c:91
void OctetType(char *Filename, char *TypeBuf)
Figure out the real type of "octet" files in case we can unarchive them.
Definition: utils.c:773
int AddToRepository(ContainerInfo *CI, char *Fuid, int Mask)
Add a ContainerInfo record to the repository AND to the database.
Definition: utils.c:1403
void FreeDirList(dirlist *DL)
Free a list of files in a directory list.
Definition: utils.c:952
int DBInsertPfile(ContainerInfo *CI, char *Fuid)
Insert a Pfile record. Sets the pfile_pk in CI.
Definition: utils.c:1115
int DBInsertUploadTree(ContainerInfo *CI, int Mask)
Insert an UploadTree record.
Definition: utils.c:1287
void Usage(char *Name, char *Version)
Display program usage.
Definition: utils.c:1724
void InitCmd()
Initialize the metahandler CMD table.
Definition: utils.c:109
int IsExe(char *Exe, int Quiet)
Check if the executable exists.
Definition: utils.c:394
void SafeExit(int rc)
Close scheduler and database connections, then exit.
Definition: utils.c:78
int FindCmd(char *Filename)
Given a file name, determine the type of extraction command. This uses Magic.
Definition: utils.c:851
int ShouldExclude(char *Filename, const char *ExcludePatterns)
Determines if a file or folder should be excluded.
Definition: utils.c:1788
int CopyFile(char *Src, char *Dst)
Copy a file. For speed: mmap and save.
Definition: utils.c:441
int IsInflatedFile(char *FileName, int InflateSize)
Test if the file is a compression bomb.
Definition: utils.c:41
char * PathCheck(char *DirPath)
Check if path contains a "%U" or "%H". If so, substitute a unique ID for U.
Definition: utils.c:1663
void SQLNoticeProcessor(void *arg, const char *message)
Dummy postgresql notice processor. This prevents Notices from being written to stderr.
Definition: utils.c:1773
dirlist * MakeDirList(char *Fullname)
Create a list of files in a directory.
Definition: utils.c:971
int ReadLine(FILE *Fin, char *Line, int MaxLine)
Read a command from a stream.
Definition: utils.c:357
BITS
File mode BITS.
Definition: utils.c:19
int IsDir(char *Fname)
Given a filename, is it a directory?
Definition: utils.c:320
char * Filename
Filename.
Definition: run_tests.c:17