copyrightUtils.cc

Go to the documentation of this file.

/*
 SPDX-FileCopyrightText: © 2014-2018,2022, Siemens AG
 Author: Daniele Fognini, Johannes Najjar
 
 SPDX-License-Identifier: GPL-2.0-only
*/
#include "copyrightUtils.hpp"
#include <boost/program_options.hpp>
 
#include <iostream>
#include <sstream>
 
using namespace std;
 
int queryAgentId(PGconn* dbConn)
{
  char* COMMIT_HASH = fo_sysconfig(AGENT_NAME, "COMMIT_HASH");
  char* VERSION = fo_sysconfig(AGENT_NAME, "VERSION");
  char* agentRevision;
  if (!asprintf(&agentRevision, "%s.%s", VERSION, COMMIT_HASH))
  {
    exit(-1);
  };
 
  int agentId = fo_GetAgentKey(dbConn,
    AGENT_NAME, 0, agentRevision, AGENT_DESC);
  free(agentRevision);
 
  if (agentId > 0)
  {
    return agentId;
  }
  else
  {
    exit(1);
  }
}
 
int writeARS(int agentId, int arsId, int uploadId, int success, const fo::DbManager& dbManager)
{
  return fo_WriteARS(dbManager.getConnection(), arsId, uploadId, agentId, AGENT_ARS, NULL, success);
}
 
void bail(int exitval)
{
  fo_scheduler_disconnect(exitval);
  exit(exitval);
}
 
bool parseCliOptions(int argc, char** argv, CliOptions& dest,
  std::vector<std::string>& fileNames, std::string& directoryToScan)
{
  unsigned type = 0;
 
  boost::program_options::options_description desc(IDENTITY ": recognized options");
  desc.add_options()
        ("help,h", "shows help")
        (
          "type,T",
          boost::program_options::value<unsigned>(&type)
            ->default_value(ALL_TYPES),
          "type of regex to try"
        ) // TODO change and add help based on IDENTITY
        (
          "verbose,v", "increase verbosity"
        )
        (
          "regex",
          boost::program_options::value<vector<string> >(),
          "user defined Regex to search: [{name=cli}@@][{matchingGroup=0}@@]{regex} e.g. 'linux@@1@@(linus) torvalds'"
        )
        (
          "files",
          boost::program_options::value< vector<string> >(),
          "files to scan"
        )
        (
          "json,J", "output JSON"
        )
        (
          "ignoreFilesWithMimeType,I", "ignoreFilesWithMimeType"
        )
        (
          "config,c", boost::program_options::value<string>(), "path to the sysconfigdir"
        )
        (
          "scheduler_start", "specifies, that the command was called by the scheduler"
        )
        (
          "userID", boost::program_options::value<int>(), "the id of the user that created the job (only in combination with --scheduler_start)"
        )
        (
          "groupID", boost::program_options::value<int>(), "the id of the group of the user that created the job (only in combination with --scheduler_start)"
        )
        (
          "jobId", boost::program_options::value<int>(), "the id of the job (only in combination with --scheduler_start)"
        )
        (
          "directory,d", boost::program_options::value<string>(), "directory to scan (recursive)"
        )
    ;
 
  boost::program_options::positional_options_description p;
  p.add("files", -1);
 
  boost::program_options::variables_map vm;
 
  try
  {
    boost::program_options::store(
      boost::program_options::command_line_parser(argc, argv).options(desc).positional(p).run(), vm);
 
    type = vm["type"].as<unsigned>();
 
    if ((vm.count("help") > 0) || (type > ALL_TYPES))
    {
      cout << desc << endl;
      exit(0);
    }
 
    if (vm.count("files"))
    {
      fileNames = vm["files"].as<std::vector<string> >();
    }
 
    unsigned long verbosity = vm.count("verbose");
    bool json = vm.count("json") > 0 ? true : false;
    bool ignoreFilesWithMimeType = vm.count("ignoreFilesWithMimeType") > 0 ? true : false;
 
    dest = CliOptions(verbosity, type, json, ignoreFilesWithMimeType);
 
    if (vm.count("regex"))
    {
      const std::vector<std::string>& userRegexesFmts = vm["regex"].as<vector<std::string> >();
      for (auto it = userRegexesFmts.begin(); it != userRegexesFmts.end(); ++it) {
        scanner* sc = makeRegexScanner(*it, "cli");
        if (!(sc))
        {
          cout << "cannot parse regex format : " << *it << endl;
          return false;
        }
        else
        {
          dest.addScanner(sc);
        }
      }
    }
 
    if (vm.count("directory"))
    {
      if (vm.count("files"))
      {
        cout << "cannot pass files and directory at the same time" << endl;
        cout << desc << endl;
        fileNames.clear();
        return false;
      }
      directoryToScan = vm["directory"].as<std::string>();
    }
 
    return true;
  }
  catch (boost::bad_any_cast&) {
    cout << "wrong parameter type" << endl;
    cout << desc << endl;
    return false;
  }
  catch (boost::program_options::error&)
  {
    cout << "wrong command line arguments" << endl;
    cout << desc << endl;
    return false;
  }
}
 
static void addDefaultScanners(CopyrightState& state)
{
  unsigned types = state.getCliOptions().getOptType();
#ifdef IDENTITY_COPYRIGHT
  if (types & 1<<0)
    //state.addMatcher(RegexMatcher(regCopyright::getType(), regCopyright::getRegex()));
    state.addScanner(new hCopyrightScanner());
 
  if (types & 1<<1)
    state.addScanner(new regexScanner("url", "copyright"));
 
  if (types & 1<<2)
    state.addScanner(new regexScanner("email", "copyright", 1));
 
  if (types & 1<<3)
    state.addScanner(new regexScanner("author", "copyright"));
#endif
 
#ifdef IDENTITY_IPRA
  if (types & 1<<0)
    state.addScanner(new regexScanner("ipra", "ipra"));
#endif
 
#ifdef IDENTITY_ECC
  if (types & 1<<0)
    state.addScanner(new regexScanner("ecc", "ecc"));
#endif
 
#ifdef IDENTITY_KW
  if (types & 1<<0)
    state.addScanner(new regexScanner("keyword", "keyword"));
#endif
}
 
scanner* makeRegexScanner(const std::string& regexDesc, const std::string& defaultType) {
  #define RGX_FMT_SEPARATOR "@@"
  auto fmtRegex = rx::regex(
    "(?:([[:alpha:]]+)" RGX_FMT_SEPARATOR ")?(?:([[:digit:]]+)" RGX_FMT_SEPARATOR ")?(.*)",
    rx::regex_constants::icase
  );
 
  rx::match_results<std::string::const_iterator> match;
  if (rx::regex_match(regexDesc.begin(), regexDesc.end(), match, fmtRegex))
  {
    std::string type(match.length(1) > 0 ? match.str(1) : defaultType.c_str());
 
    int regId = match.length(2) > 0 ? std::stoi(std::string(match.str(2))) : 0;
 
    if (match.length(3) == 0)
      return 0; // nullptr
 
    std::istringstream stream;
    stream.str(type + "=" + match.str(3));
    return new regexScanner(type, stream, regId);
  }
  return 0; // nullptr
}
 
CopyrightState getState(CliOptions&& cliOptions)
{
  CopyrightState state(std::move(cliOptions));
  addDefaultScanners(state);
 
  return state;
}
 
bool saveToDatabase(const string& s, const list<match>& matches, unsigned long pFileId, int agentId, const CopyrightDatabaseHandler& copyrightDatabaseHandler)
{
  if (!copyrightDatabaseHandler.begin())
  {
    return false;
  }
 
  size_t count = 0;
  for (auto m = matches.begin(); m != matches.end(); ++m)
  {
 
    DatabaseEntry entry;
    entry.agent_fk = agentId;
    entry.content = cleanMatch(s, *m);
    entry.copy_endbyte = m->end;
    entry.copy_startbyte = m->start;
    entry.pfile_fk = pFileId;
    entry.type = m->type;
 
    if (entry.content.length() != 0)
    {
      ++count;
      if (!copyrightDatabaseHandler.insertInDatabase(entry))
      {
        copyrightDatabaseHandler.rollback();
        return false;
      };
    }
  }
 
  return copyrightDatabaseHandler.commit();
}
 
void matchFileWithLicenses(const string& sContent, unsigned long pFileId, CopyrightState const& state, int agentId, CopyrightDatabaseHandler& databaseHandler)
{
  list<match> l;
  const list<unptr::shared_ptr<scanner>>& scanners = state.getScanners();
  for (auto sc = scanners.begin(); sc != scanners.end(); ++sc)
  {
    (*sc)->ScanString(sContent, l);
  }
  saveToDatabase(sContent, l, pFileId, agentId, databaseHandler);
}
 
void matchPFileWithLicenses(CopyrightState const& state, int agentId, unsigned long pFileId, CopyrightDatabaseHandler& databaseHandler)
{
  char* pFile = databaseHandler.getPFileNameForFileId(pFileId);
 
  if (!pFile)
  {
    cout << "File not found " << pFileId << endl;
    bail(8);
  }
 
  char* fileName = NULL;
  {
#pragma omp critical (repo_mk_path)
    fileName = fo_RepMkPath("files", pFile);
  }
  if (fileName)
  {
    string s;
    ReadFileToString(fileName, s);
 
    matchFileWithLicenses(s, pFileId, state, agentId, databaseHandler);
 
    free(fileName);
    free(pFile);
  }
  else
  {
    cout << "PFile not found in repo " << pFileId << endl;
    bail(7);
  }
}
 
bool processUploadId(const CopyrightState& state, int agentId, int uploadId, CopyrightDatabaseHandler& databaseHandler, bool ignoreFilesWithMimeType)
{
  vector<unsigned long> fileIds = databaseHandler.queryFileIdsForUpload(agentId, uploadId, ignoreFilesWithMimeType);
 
#pragma omp parallel num_threads(THREADS)
  {
    CopyrightDatabaseHandler threadLocalDatabaseHandler(databaseHandler.spawn());
 
    size_t pFileCount = fileIds.size();
#pragma omp for
    for (size_t it = 0; it < pFileCount; ++it)
    {
      unsigned long pFileId = fileIds[it];
 
      if (pFileId == 0)
      {
        continue;
      }
 
      matchPFileWithLicenses(state, agentId, pFileId, threadLocalDatabaseHandler);
 
      fo_scheduler_heart(1);
    }
  }
 
  return true;
}
 
pair<string, list<match>> processSingleFile(const CopyrightState& state,
  const string fileName)
{
  const list<unptr::shared_ptr<scanner>>& scanners = state.getScanners();
  list<match> matchList;
 
  // Read file into one string
  string s;
  if (!ReadFileToString(fileName, s))
  {
    // File error
    s = "";
  }
  else
  {
    for (auto sc = scanners.begin(); sc != scanners.end(); ++sc)
    {
      (*sc)->ScanString(s, matchList);
    }
  }
  return make_pair(s, matchList);
}
 
void appendToJson(const std::string fileName,
    const std::pair<string, list<match>> resultPair, bool &printComma)
{
  Json::Value result;
#if JSONCPP_VERSION_HEXA < ((1 << 24) | (4 << 16))
  // Use FastWriter for versions below 1.4.0
  Json::FastWriter jsonWriter;
#else
  // Since version 1.4.0, FastWriter is deprecated and replaced with
  // StreamWriterBuilder
  Json::StreamWriterBuilder jsonWriter;
  jsonWriter["commentStyle"] = "None";
  jsonWriter["indentation"] = "";
#endif
 
  if (resultPair.first.empty())
  {
    result["file"] = fileName;
    result["results"] = "Unable to read file";
  }
  else
  {
    list<match> resultList = resultPair.second;
    Json::Value results;
    for (auto m : resultList)
    {
      Json::Value j;
      j["start"] = m.start;
      j["end"] = m.end;
      j["type"] = m.type;
      j["content"] = cleanMatch(resultPair.first, m);
      results.append(j);
    }
    result["file"] = fileName;
    result["results"] = results;
  }
  // Thread-Safety: output all matches JSON at once to STDOUT
#pragma omp critical (jsonPrinter)
  {
    if (printComma)
    {
      cout << "," << endl;
    }
    else
    {
      printComma = true;
    }
    string jsonString;
#if JSONCPP_VERSION_HEXA < ((1 << 24) | (4 << 16))
    // For version below 1.4.0, every writer append `\n` at end.
    // Find and replace it.
    jsonString = jsonWriter.write(result);
    jsonString.replace(jsonString.find("\n"), string("\n").length(), "");
#else
    // For version >= 1.4.0, \n is not appended.
    jsonString = Json::writeString(jsonWriter, result);
#endif
    cout << "  " << jsonString << flush;
  }
}
 
void printResultToStdout(const std::string fileName,
    const std::pair<string, list<match>> resultPair)
{
  if (resultPair.first.empty())
  {
    cout << fileName << " :: Unable to read file" << endl;
    return;
  }
  stringstream ss;
  ss << fileName << " ::" << endl;
  // Output matches
  list<match> resultList = resultPair.second;
  for (auto m = resultList.begin();  m != resultList.end(); ++m)
  {
    ss << "\t[" << m->start << ':' << m->end << ':' << m->type << "] '"
       << cleanMatch(resultPair.first, *m)
       << "'" << endl;
  }
  // Thread-Safety: output all matches (collected in ss) at once to cout
  cout << ss.str();
}