11 #include "nomos_utils.h"
15 extern int should_connect_to_db;
42 char insert[myBUFSIZ];
43 char escLicName[myBUFSIZ];
44 char *specialLicenseText;
52 len = strlen(licenseName);
53 PQescapeStringConn(gl.
pgConn, escLicName, licenseName, len, &error);
55 LOG_WARNING(
"Does license name %s have multibyte encoding?", licenseName)
58 snprintf(query, myBUFSIZ - 1,
"SELECT rf_pk FROM " LICENSE_REF_TABLE
" where rf_shortname='%s'", escLicName);
59 result = PQexec(gl.
pgConn, query);
62 numRows = PQntuples(result);
65 rf_pk = atol(PQgetvalue(result, 0, 0));
72 specialLicenseText =
"License by Nomos.";
74 snprintf(insert, myBUFSIZ - 1,
"insert into license_ref(rf_shortname, rf_text, rf_detector_type) values('%s', '%s', 2)", escLicName,
76 result = PQexec(gl.
pgConn, insert);
79 || ((PQresultStatus(result) != PGRES_COMMAND_OK)
80 && (strncmp(PG_ERRCODE_UNIQUE_VIOLATION, PQresultErrorField(result, PG_DIAG_SQLSTATE), 5))))
82 printf(
"ERROR: %s(%d): Nomos failed to add a new license. %s/n: %s/n",
83 __FILE__, __LINE__, PQresultErrorMessage(result), insert);
90 result = PQexec(gl.
pgConn, query);
93 numRows = PQntuples(result);
95 rf_pk = atol(PQgetvalue(result, 0, 0));
98 printf(
"ERROR: %s:%s:%d Just inserted value is missing. On: %s", __FILE__,
"add2license_ref()", __LINE__, query);
122 len = (strlen(rf_shortname) <
sizeof(long)) ? strlen(rf_shortname) :
sizeof(long);
123 for (i = 0; i < len; i++)
124 hashval += rf_shortname[i] << 8 * i;
125 hashval = hashval % pcroot->
maxnodes;
142 pcnode = pcroot->
nodes;
143 for (i = 0; i < pcroot->
maxnodes; i++)
145 if (pcnode->
rf_pk != 0L)
166 pcnode = pcroot->
nodes;
167 for (i = 0; i < pcroot->
maxnodes; i++)
169 if (pcnode->
rf_pk != 0L)
198 for (i = 0; i < pcroot->
maxnodes; i++)
200 noden = (hashval + i) & (pcroot->
maxnodes - 1);
202 pcnode = pcroot->
nodes + noden;
206 pcnode->
rf_pk = rf_pk;
210 if (i < pcroot->maxnodes)
235 for (i = 0; i < pcroot->
maxnodes; i++)
237 noden = (hashval + i) & (pcroot->
maxnodes - 1);
239 pcnode = pcroot->
nodes + noden;
244 return pcnode->
rf_pk;
267 char query[myBUFSIZ];
274 sprintf(query,
"SELECT rf_pk, rf_shortname FROM " LICENSE_REF_TABLE
" where rf_detector_type=2");
275 result = PQexec(gl.
pgConn, query);
279 numLics = PQntuples(result);
281 for (row = 0; row < numLics; row++)
283 lrcache_add(pcroot, atol(PQgetvalue(result, row, 0)), PQgetvalue(result, row, 1));
308 if ((len = strlen(rf_shortname)) == 0)
310 printf(
"ERROR! Nomos.c get_rfpk() passed empty name");
337 char *value,
int valueMax,
char separator)
345 traceFunc(
"== getFieldValue(inStr= %s fieldMax= %d separator= '%c'\n",
346 inStr, fieldMax, separator);
349 memset(field, 0, fieldMax);
350 memset(value, 0, valueMax);
353 while (isspace(inStr[0]))
358 if (inStr[0] ==
'\0')
366 for (
s = 0; (inStr[
s] !=
'\0') && !isspace(inStr[
s]) && (inStr[
s] !=
'=');
s++)
368 field[f++] = inStr[
s];
372 while (isspace(inStr[
s]))
377 if (inStr[
s] != separator)
381 if (inStr[
s] ==
'\0')
389 while (isspace(inStr[
s]))
393 if (inStr[
s] ==
'\0')
399 if ((inStr[
s] ==
'\'') || (inStr[
s] ==
'"'))
403 if (inStr[
s] ==
'\0')
411 for (; (inStr[
s] !=
'\0') && (inStr[
s] != gotQuote);
s++)
413 if (inStr[
s] ==
'\\')
415 value[v++] = inStr[++
s];
419 value[v++] = inStr[
s];
426 for (; (inStr[
s] !=
'\0') && !isspace(inStr[
s]);
s++)
428 if (inStr[
s] ==
'\\')
430 value[v++] = inStr[++
s];
434 value[v++] = inStr[
s];
439 while (isspace(inStr[
s]))
462 if ((strlen(cur.
compLic)) == 0)
468 if (strstr(cur.
compLic,
",") == NULL)
477 cur.
tmpLics = strtok_r(saveLicsPtr,
",", &saveptr);
485 cur.
tmpLics = strtok_r(saveLicsPtr,
",", &saveptr);
515 should_connect_to_db = 0;
516 printf(
"Usage: %s [options] [file [file [...]]\n", Name);
517 printf(
" -h :: help (print this message), then exit.\n");
518 printf(
" -i :: initialize the database, then exit.\n");
519 printf(
" -c :: specify the directory for the system configuration.\n");
520 printf(
" -l :: print full file path (command line only).\n");
521 printf(
" -v :: verbose (-vv = more verbose)\n");
522 printf(
" -J :: output in JSON\n");
523 printf(
" -S :: print Highlightinfo to stdout \n");
524 printf(
" file :: if files are listed, print the licenses detected within them.\n");
525 printf(
" no file :: process data from the scheduler.\n");
526 printf(
" -V :: print the version info, then exit.\n");
527 printf(
" -d :: specify a directory to scan.\n");
528 printf(
" -n :: spaw n - 1 child processes to run, there will be n running processes(the parent and n - 1 children). \n the default n is 2(when n is less than 2 or not setting, will be changed to 2) when -d is specified.\n");
538 FUNCTION
void Bail(
int exitval)
541 traceFunc(
"== Bail(%d)\n", exitval);
544 #if defined(MEMORY_TRACING) && defined(MEM_ACCT)
547 memCacheDump(
"Mem-cache @ Bail() time:");
570 traceFunc(
"== optionIsSet(%x)\n", val);
590 traceFunc(
"== getFileLists(%s)\n", dirpath);
594 listInit(&cur.regfList, 0,
"regular-files list");
595 listInit(&cur.offList, 0,
"buffer-offset list");
596 #ifdef FLAG_NO_COPYRIGHT
597 listInit(&gl.nocpyrtList, 0,
"no-copyright list");
630 fo_dbManager_PrepareStamement(
633 "INSERT INTO license_file(rf_fk, agent_fk, pfile_fk) VALUES($1, $2, $3) RETURNING fl_pk",
640 long licenseFileId = -1;
641 if (PQntuples(result) > 0) {
642 licenseFileId = atol(PQgetvalue(result, 0, 0));
646 return (licenseFileId);
664 if ((index >= _KW_first) && (index <= _KW_last))
666 else if (index > _KW_last)
701 PGresult* begin1 = PQexec(gl.
pgConn,
"BEGIN");
706 preparedKeywords = fo_dbManager_PrepareStamement(
708 "updateLicenseHighlighting:keyword",
709 "INSERT INTO highlight_keyword (pfile_fk, start, len) VALUES($1, $2, $3)",
717 #pragma GCC diagnostic push
718 #pragma GCC diagnostic ignored "-Wmaybe-uninitialized"
723 #pragma GCC diagnostic pop
729 PGresult* commit1 = PQexec(gl.
pgConn,
"COMMIT");
732 PGresult* begin2 =PQexec(gl.
pgConn,
"BEGIN");
737 preparedLicenses=fo_dbManager_PrepareStamement(
739 "updateLicenseHighlighting",
740 "INSERT INTO highlight (fl_fk, start, len, type) VALUES($1, $2, $3,'L')",
757 #pragma GCC diagnostic push
758 #pragma GCC diagnostic ignored "-Wmaybe-uninitialized"
765 #pragma GCC diagnostic pop
775 PGresult* commit2 = PQexec(gl.
pgConn,
"COMMIT");
791 traceFunc(
"== processFile(%s)\n", fileToScan);
799 pathcopy = g_strdup(fileToScan);
800 strcpy(cur.
targetDir, dirname(pathcopy));
807 LOG_FATAL(
"\"%s\" is not a plain file", fileToScan)
812 listInit(&cur.fLicFoundMap, 0,
"file-license-found map");
813 listInit(&cur.parseList, 0,
"license-components list");
814 listInit(&cur.lList, 0,
"license-list");
830 if (strcmp(licenseName, ourLicence->
licenceName) == 0)
842 long rf_pk =
get_rfpk(pcroot, licenseName);
844 if (licenseFileId > 0) {
878 printf(
"%s\n",scanRecord->
compLic);
883 if (noneFound != NULL)
893 for (numLicenses = 0; cur.
licenseList[numLicenses] != NULL; numLicenses++)
901 printf(
"Failure in update of highlight table \n");
926 GArray* in,
int index)
940 cur->
indexList = g_array_new(FALSE, FALSE,
sizeof(
int));
943 cur->docBufferPositionsAndOffsets = g_array_new(FALSE, FALSE,
sizeof(
pairPosOff));
944 cur->currentLicenceIndex=-1;
958 listClear(&thisScan->regfList, DEALLOC_LIST);
959 listClear(&thisScan->offList, DEALLOC_LIST);
960 listClear(&thisScan->fLicFoundMap, DEALLOC_LIST);
961 listClear(&thisScan->parseList, DEALLOC_LIST);
962 listClear(&thisScan->lList, DEALLOC_LIST);
966 g_array_free(thisScan->docBufferPositionsAndOffsets, TRUE);
981 for(i=0; i< theMatches->len; ++i) {
984 g_array_free( theMatches, TRUE);
993 if(in->licenceName) g_free(in->licenceName);
994 g_array_free(in->matchPositions, TRUE);
995 g_array_free(in->indexList,TRUE);
1003 FUNCTION
inline void addLicence(GArray* theMatches,
char* licenceName ) {
1006 cur.
indexList=g_array_new(FALSE, FALSE,
sizeof(
int));
1012 g_array_append_val(theMatches , newMatch);
int s
The socket that the CLI will use to communicate.
int fo_checkPQresult(PGconn *pgConn, PGresult *result, char *sql, char *FileID, int LineNumb)
Check the result status of a postgres SELECT.
void fo_scheduler_disconnect(int retcode)
Disconnect the scheduler connection.
item_t * listGetItem(list_t *l, char *s)
get an item from the itemlist. If the item is not in the itemlist, then add it to the itemlist.
void listInit(list_t *l, int size, char *label)
intialize a list, if the list is not empty, empty it (initialize it to zero's).
void listClear(list_t *l, int deallocFlag)
Destroy list_t.
PGresult * fo_dbManager_ExecPrepared(fo_dbManager_PreparedStatement *preparedStatement,...)
Execute a prepared statement.
void fo_dbManager_free(fo_dbManager *dbManager)
Un-allocate the memory from a DB manager.
int isFILE(char *pathname)
Check if an inode is a file.
FUNCTION void lrcache_print(cacheroot_t *pcroot)
Print the contents of the hash table.
FUNCTION void freeAndClearScan(struct curScan *thisScan)
Clean-up all the per scan data structures, freeing any old data.
FUNCTION void lrcache_free(cacheroot_t *pcroot)
free the hash table
FUNCTION char * getFieldValue(char *inStr, char *field, int fieldMax, char *value, int valueMax, char separator)
Given a string that contains field='value' pairs, save the items.
FUNCTION long lrcache_lookup(cacheroot_t *pcroot, char *rf_shortname)
lookup rf_pk in the license_ref cache rf_shortname is the key
FUNCTION void Usage(char *Name)
Print nomos usage help.
sem_t * mutexJson
Mutex to handle JSON writes.
FUNCTION void processFile(char *fileToScan)
process a single file
FUNCTION void cleanTheMatches(GArray *theMatches)
Cleans the match array and free the memory.
FUNCTION void parseLicenseList()
parse the comma separated list of license names found
FUNCTION long updateLicenseFile(long rfPk)
insert rf_fk, agent_fk and pfile_fk into license_file table
FUNCTION void initializeCurScan(struct curScan *cur)
Initialize the scanner.
FUNCTION int recordScanToDB(cacheroot_t *pcroot, struct curScan *scanRecord)
Write out the information about the scan to the FOSSology database.
FUNCTION int lrcache_add(cacheroot_t *pcroot, long rf_pk, char *rf_shortname)
add a rf_shortname, rf_pk to the license_ref cache rf_shortname is the key
void cleanLicenceBuffer()
Clean the license buffer.
FUNCTION void addLicence(GArray *theMatches, char *licenceName)
Add a license to the matches array.
FUNCTION MatchPositionAndType * getMatchfromHighlightInfo(GArray *in, int index)
Get the MatchPositionAndType for a given index in highlight array.
FUNCTION void getFileLists(char *dirpath)
Initialize the lists: regular-files list cur.regfList and buffer-offset list cur.offList.
FUNCTION LicenceAndMatchPositions * getLicenceAndMatchPositions(GArray *in, int index)
Get the LicenceAndMatchPositions for a given index in match array.
gboolean * printcomma
True to print comma while printing JSON object.
FUNCTION void cleanLicenceAndMatchPositions(LicenceAndMatchPositions *in)
Cleans the license and match positions object and free the memory.
FUNCTION int initLicRefCache(cacheroot_t *pcroot)
build a cache the license ref db table.
char saveLics[myBUFSIZ]
License string.
int updateLicenseFileAndHighlightArray(char *licenseName, cacheroot_t *pcroot)
Add a license to hash table, license table and highlight array.
FUNCTION int optionIsSet(int val)
Check if an CLI option is set.
bool clearLastElementOfLicenceBuffer()
Remove the last element from license buffer.
void setLicenseFileIdInHiglightArray(long licenseFileId, char *licenseName)
Set the license file id to the highlights.
FUNCTION long lrcache_hash(cacheroot_t *pcroot, char *rf_shortname)
calculate the hash of an rf_shortname rf_shortname is the key
FUNCTION void Bail(int exitval)
Close connections and exit.
FUNCTION char convertIndexToHighlightType(int index)
Return the highlight type (K|L|0) for a given index.
FUNCTION long get_rfpk(cacheroot_t *pcroot, char *rf_shortname)
Get the rf_pk for rf_shortname.
FUNCTION long add2license_ref(char *licenseName)
Add a new license to license_ref table.
FUNCTION int updateLicenseHighlighting(cacheroot_t *pcroot)
insert rf_fk, agent_fk, offset, len and type into highlight table
GArray * matchPositions
Match positions.
GArray * indexList
License indexes.
char * licenceName
License names.
int licenseFileId
PFile id.
int start
Start position of match.
int end
End position of match.
long rf_pk
License id from database.
char * rf_shortname
License shortname.
int maxnodes
No. of nodes in the list.
cachenode_t * nodes
Array of nodes.
Struct that tracks state related to current file being scanned.
GArray * keywordPositions
char targetFile[myBUFSIZ]
PGconn * pgConn
DB Connection.
fo_dbManager * dbManager
FOSSology DB manager.
char initwd[myBUFSIZ]
CDB, would like to workaround/eliminate.