11 #include "nomos_utils.h"
14 extern int should_connect_to_db;
41 char insert[myBUFSIZ];
42 char escLicName[myBUFSIZ];
43 char *specialLicenseText;
51 len = strlen(licenseName);
52 PQescapeStringConn(gl.
pgConn, escLicName, licenseName, len, &error);
54 LOG_WARNING(
"Does license name %s have multibyte encoding?", licenseName)
57 snprintf(query, myBUFSIZ - 1,
"SELECT rf_pk FROM " LICENSE_REF_TABLE
" where rf_shortname='%s'", escLicName);
58 result = PQexec(gl.
pgConn, query);
61 numRows = PQntuples(result);
64 rf_pk = atol(PQgetvalue(result, 0, 0));
71 specialLicenseText =
"License by Nomos.";
73 snprintf(insert, myBUFSIZ - 1,
"insert into license_ref(rf_shortname, rf_text, rf_detector_type) values('%s', '%s', 2)", escLicName,
75 result = PQexec(gl.
pgConn, insert);
78 || ((PQresultStatus(result) != PGRES_COMMAND_OK)
79 && (strncmp(PG_ERRCODE_UNIQUE_VIOLATION, PQresultErrorField(result, PG_DIAG_SQLSTATE), 5))))
81 printf(
"ERROR: %s(%d): Nomos failed to add a new license. %s/n: %s/n",
82 __FILE__, __LINE__, PQresultErrorMessage(result), insert);
89 result = PQexec(gl.
pgConn, query);
92 numRows = PQntuples(result);
94 rf_pk = atol(PQgetvalue(result, 0, 0));
97 printf(
"ERROR: %s:%s:%d Just inserted value is missing. On: %s", __FILE__,
"add2license_ref()", __LINE__, query);
121 len = (strlen(rf_shortname) <
sizeof(long)) ? strlen(rf_shortname) :
sizeof(long);
122 for (i = 0; i < len; i++)
123 hashval += rf_shortname[i] << 8 * i;
124 hashval = hashval % pcroot->
maxnodes;
141 pcnode = pcroot->
nodes;
142 for (i = 0; i < pcroot->
maxnodes; i++)
144 if (pcnode->
rf_pk != 0L)
165 pcnode = pcroot->
nodes;
166 for (i = 0; i < pcroot->
maxnodes; i++)
168 if (pcnode->
rf_pk != 0L)
197 for (i = 0; i < pcroot->
maxnodes; i++)
199 noden = (hashval + i) & (pcroot->
maxnodes - 1);
201 pcnode = pcroot->
nodes + noden;
205 pcnode->
rf_pk = rf_pk;
209 if (i < pcroot->maxnodes)
234 for (i = 0; i < pcroot->
maxnodes; i++)
236 noden = (hashval + i) & (pcroot->
maxnodes - 1);
238 pcnode = pcroot->
nodes + noden;
243 return pcnode->
rf_pk;
266 char query[myBUFSIZ];
273 sprintf(query,
"SELECT rf_pk, rf_shortname FROM " LICENSE_REF_TABLE
" where rf_detector_type=2");
274 result = PQexec(gl.
pgConn, query);
278 numLics = PQntuples(result);
280 for (row = 0; row < numLics; row++)
282 lrcache_add(pcroot, atol(PQgetvalue(result, row, 0)), PQgetvalue(result, row, 1));
307 if ((len = strlen(rf_shortname)) == 0)
309 printf(
"ERROR! Nomos.c get_rfpk() passed empty name");
336 char *value,
int valueMax,
char separator)
344 traceFunc(
"== getFieldValue(inStr= %s fieldMax= %d separator= '%c'\n",
345 inStr, fieldMax, separator);
348 memset(field, 0, fieldMax);
349 memset(value, 0, valueMax);
352 while (isspace(inStr[0]))
357 if (inStr[0] ==
'\0')
365 for (
s = 0; (inStr[
s] !=
'\0') && !isspace(inStr[
s]) && (inStr[
s] !=
'=');
s++)
367 field[f++] = inStr[
s];
371 while (isspace(inStr[
s]))
376 if (inStr[
s] != separator)
380 if (inStr[
s] ==
'\0')
388 while (isspace(inStr[
s]))
392 if (inStr[
s] ==
'\0')
398 if ((inStr[
s] ==
'\'') || (inStr[
s] ==
'"'))
402 if (inStr[
s] ==
'\0')
410 for (; (inStr[
s] !=
'\0') && (inStr[
s] != gotQuote);
s++)
412 if (inStr[
s] ==
'\\')
414 value[v++] = inStr[++
s];
418 value[v++] = inStr[
s];
425 for (; (inStr[
s] !=
'\0') && !isspace(inStr[
s]);
s++)
427 if (inStr[
s] ==
'\\')
429 value[v++] = inStr[++
s];
433 value[v++] = inStr[
s];
438 while (isspace(inStr[
s]))
461 if ((strlen(cur.
compLic)) == 0)
467 if (strstr(cur.
compLic,
",") == NULL)
476 cur.
tmpLics = strtok_r(saveLicsPtr,
",", &saveptr);
484 cur.
tmpLics = strtok_r(saveLicsPtr,
",", &saveptr);
514 should_connect_to_db = 0;
515 printf(
"Usage: %s [options] [file [file [...]]\n", Name);
516 printf(
" -h :: help (print this message), then exit.\n");
517 printf(
" -i :: initialize the database, then exit.\n");
518 printf(
" -c :: specify the directory for the system configuration.\n");
519 printf(
" -l :: print full file path (command line only).\n");
520 printf(
" -v :: verbose (-vv = more verbose)\n");
521 printf(
" -J :: output in JSON\n");
522 printf(
" -S :: print Highlightinfo to stdout \n");
523 printf(
" file :: if files are listed, print the licenses detected within them.\n");
524 printf(
" no file :: process data from the scheduler.\n");
525 printf(
" -V :: print the version info, then exit.\n");
526 printf(
" -d :: specify a directory to scan.\n");
527 printf(
" -n :: spaw n - 1 child processes to run, there will be n running processes(the parent and n - 1 children). \n the default n is 2(when n is less than 2 or not setting, will be changed to 2) when -d is specified.\n");
537 FUNCTION
void Bail(
int exitval)
540 traceFunc(
"== Bail(%d)\n", exitval);
543 #if defined(MEMORY_TRACING) && defined(MEM_ACCT)
546 memCacheDump(
"Mem-cache @ Bail() time:");
567 traceFunc(
"== optionIsSet(%x)\n", val);
587 traceFunc(
"== getFileLists(%s)\n", dirpath);
591 listInit(&cur.regfList, 0,
"regular-files list");
592 listInit(&cur.offList, 0,
"buffer-offset list");
593 #ifdef FLAG_NO_COPYRIGHT
594 listInit(&gl.nocpyrtList, 0,
"no-copyright list");
627 fo_dbManager_PrepareStamement(
630 "INSERT INTO license_file(rf_fk, agent_fk, pfile_fk) VALUES($1, $2, $3) RETURNING fl_pk",
637 long licenseFileId = -1;
638 if (PQntuples(result) > 0) {
639 licenseFileId = atol(PQgetvalue(result, 0, 0));
643 return (licenseFileId);
661 if ((index >= _KW_first) && (index <= _KW_last))
663 else if (index > _KW_last)
698 PGresult* begin1 = PQexec(gl.
pgConn,
"BEGIN");
703 preparedKeywords = fo_dbManager_PrepareStamement(
705 "updateLicenseHighlighting:keyword",
706 "INSERT INTO highlight_keyword (pfile_fk, start, len) VALUES($1, $2, $3)",
714 #pragma GCC diagnostic push
715 #pragma GCC diagnostic ignored "-Wmaybe-uninitialized"
720 #pragma GCC diagnostic pop
726 PGresult* commit1 = PQexec(gl.
pgConn,
"COMMIT");
729 PGresult* begin2 =PQexec(gl.
pgConn,
"BEGIN");
734 preparedLicenses=fo_dbManager_PrepareStamement(
736 "updateLicenseHighlighting",
737 "INSERT INTO highlight (fl_fk, start, len, type) VALUES($1, $2, $3,'L')",
754 #pragma GCC diagnostic push
755 #pragma GCC diagnostic ignored "-Wmaybe-uninitialized"
762 #pragma GCC diagnostic pop
772 PGresult* commit2 = PQexec(gl.
pgConn,
"COMMIT");
788 traceFunc(
"== processFile(%s)\n", fileToScan);
796 pathcopy = g_strdup(fileToScan);
797 strcpy(cur.
targetDir, dirname(pathcopy));
804 LOG_FATAL(
"\"%s\" is not a plain file", fileToScan)
809 listInit(&cur.fLicFoundMap, 0,
"file-license-found map");
810 listInit(&cur.parseList, 0,
"license-components list");
811 listInit(&cur.lList, 0,
"license-list");
827 if (strcmp(licenseName, ourLicence->
licenceName) == 0)
839 long rf_pk =
get_rfpk(pcroot, licenseName);
841 if (licenseFileId > 0) {
875 printf(
"%s\n",scanRecord->
compLic);
880 if (noneFound != NULL)
890 for (numLicenses = 0; cur.
licenseList[numLicenses] != NULL; numLicenses++)
898 printf(
"Failure in update of highlight table \n");
923 GArray* in,
int index)
937 cur->
indexList = g_array_new(FALSE, FALSE,
sizeof(
int));
940 cur->docBufferPositionsAndOffsets = g_array_new(FALSE, FALSE,
sizeof(
pairPosOff));
941 cur->currentLicenceIndex=-1;
955 listClear(&thisScan->regfList, DEALLOC_LIST);
956 listClear(&thisScan->offList, DEALLOC_LIST);
957 listClear(&thisScan->fLicFoundMap, DEALLOC_LIST);
958 listClear(&thisScan->parseList, DEALLOC_LIST);
959 listClear(&thisScan->lList, DEALLOC_LIST);
963 g_array_free(thisScan->docBufferPositionsAndOffsets, TRUE);
978 for(i=0; i< theMatches->len; ++i) {
981 g_array_free( theMatches, TRUE);
990 if(in->licenceName) g_free(in->licenceName);
991 g_array_free(in->matchPositions, TRUE);
992 g_array_free(in->indexList,TRUE);
1000 FUNCTION
inline void addLicence(GArray* theMatches,
char* licenceName ) {
1003 cur.
indexList=g_array_new(FALSE, FALSE,
sizeof(
int));
1009 g_array_append_val(theMatches , newMatch);
int s
The socket that the CLI will use to communicate.
int fo_checkPQresult(PGconn *pgConn, PGresult *result, char *sql, char *FileID, int LineNumb)
Check the result status of a postgres SELECT.
void fo_scheduler_disconnect(int retcode)
Disconnect the scheduler connection.
item_t * listGetItem(list_t *l, char *s)
get an item from the itemlist. If the item is not in the itemlist, then add it to the itemlist.
void listInit(list_t *l, int size, char *label)
intialize a list, if the list is not empty, empty it (initialize it to zero's).
void listClear(list_t *l, int deallocFlag)
Destroy list_t.
PGresult * fo_dbManager_ExecPrepared(fo_dbManager_PreparedStatement *preparedStatement,...)
Execute a prepared statement.
void fo_dbManager_free(fo_dbManager *dbManager)
Un-allocate the memory from a DB manager.
int isFILE(char *pathname)
Check if an inode is a file.
FUNCTION void lrcache_print(cacheroot_t *pcroot)
Print the contents of the hash table.
FUNCTION void freeAndClearScan(struct curScan *thisScan)
Clean-up all the per scan data structures, freeing any old data.
FUNCTION void lrcache_free(cacheroot_t *pcroot)
free the hash table
FUNCTION char * getFieldValue(char *inStr, char *field, int fieldMax, char *value, int valueMax, char separator)
Given a string that contains field='value' pairs, save the items.
FUNCTION long lrcache_lookup(cacheroot_t *pcroot, char *rf_shortname)
lookup rf_pk in the license_ref cache rf_shortname is the key
FUNCTION void Usage(char *Name)
Print nomos usage help.
sem_t * mutexJson
Mutex to handle JSON writes.
FUNCTION void processFile(char *fileToScan)
process a single file
FUNCTION void cleanTheMatches(GArray *theMatches)
Cleans the match array and free the memory.
FUNCTION void parseLicenseList()
parse the comma separated list of license names found
FUNCTION long updateLicenseFile(long rfPk)
insert rf_fk, agent_fk and pfile_fk into license_file table
FUNCTION void initializeCurScan(struct curScan *cur)
Initialize the scanner.
FUNCTION int recordScanToDB(cacheroot_t *pcroot, struct curScan *scanRecord)
Write out the information about the scan to the FOSSology database.
FUNCTION int lrcache_add(cacheroot_t *pcroot, long rf_pk, char *rf_shortname)
add a rf_shortname, rf_pk to the license_ref cache rf_shortname is the key
void cleanLicenceBuffer()
Clean the license buffer.
FUNCTION void addLicence(GArray *theMatches, char *licenceName)
Add a license to the matches array.
FUNCTION MatchPositionAndType * getMatchfromHighlightInfo(GArray *in, int index)
Get the MatchPositionAndType for a given index in highlight array.
FUNCTION void getFileLists(char *dirpath)
Initialize the lists: regular-files list cur.regfList and buffer-offset list cur.offList.
FUNCTION LicenceAndMatchPositions * getLicenceAndMatchPositions(GArray *in, int index)
Get the LicenceAndMatchPositions for a given index in match array.
gboolean * printcomma
True to print comma while printing JSON object.
FUNCTION void cleanLicenceAndMatchPositions(LicenceAndMatchPositions *in)
Cleans the license and match positions object and free the memory.
FUNCTION int initLicRefCache(cacheroot_t *pcroot)
build a cache the license ref db table.
char saveLics[myBUFSIZ]
License string.
int updateLicenseFileAndHighlightArray(char *licenseName, cacheroot_t *pcroot)
Add a license to hash table, license table and highlight array.
FUNCTION int optionIsSet(int val)
Check if an CLI option is set.
bool clearLastElementOfLicenceBuffer()
Remove the last element from license buffer.
void setLicenseFileIdInHiglightArray(long licenseFileId, char *licenseName)
Set the license file id to the highlights.
FUNCTION long lrcache_hash(cacheroot_t *pcroot, char *rf_shortname)
calculate the hash of an rf_shortname rf_shortname is the key
FUNCTION void Bail(int exitval)
Close connections and exit.
FUNCTION char convertIndexToHighlightType(int index)
Return the highlight type (K|L|0) for a given index.
FUNCTION long get_rfpk(cacheroot_t *pcroot, char *rf_shortname)
Get the rf_pk for rf_shortname.
FUNCTION long add2license_ref(char *licenseName)
Add a new license to license_ref table.
FUNCTION int updateLicenseHighlighting(cacheroot_t *pcroot)
insert rf_fk, agent_fk, offset, len and type into highlight table
GArray * matchPositions
Match positions.
GArray * indexList
License indexes.
char * licenceName
License names.
int licenseFileId
PFile id.
int start
Start position of match.
int end
End position of match.
long rf_pk
License id from database.
char * rf_shortname
License shortname.
int maxnodes
No. of nodes in the list.
cachenode_t * nodes
Array of nodes.
Struct that tracks state related to current file being scanned.
GArray * keywordPositions
char targetFile[myBUFSIZ]
PGconn * pgConn
DB Connection.
fo_dbManager * dbManager
FOSSology DB manager.
char initwd[myBUFSIZ]
CDB, would like to workaround/eliminate.