CustomTextImport.php

Go to the documentation of this file.

<?php
/*
 SPDX-FileCopyrightText: © 2025 Harshit Gandhi <gandhiharshit716@gmail.com>
 SPDX-FileCopyrightText: © Fossology contributors
 
 SPDX-License-Identifier: GPL-2.0-only
*/
 
namespace Fossology\Lib\Application;
use Fossology\Lib\Auth\Auth;
 
use Fossology\Lib\Dao\LicenseDao;
use Fossology\Lib\Dao\UserDao;
use Fossology\Lib\Db\DbManager;
use Fossology\Lib\Util\ArrayOperation;
use Exception;
 
class CustomTextImport
{
  protected $dbManager;
  protected $userDao;
  protected $licenseDao;
  protected $delimiter = ',';
  protected $enclosure = '"';
  protected $headrow = null;
  protected $alias = array(
      'text'=>array('text','Text'),
      'acknowledgement'=>array('acknowledgement','Acknowledgement','acknowledgements'),
      'comments'=>array('comments','Comments'),
      'is_active'=>array('is_active','Is Active','active'),
      'created_by'=>array('created_by','Created By','user_name'),
      'group'=>array('group','Group','group_name'),
      'licenses_to_add'=>array('licenses_to_add','Licenses To Add','add_licenses'),
      'licenses_to_remove'=>array('licenses_to_remove','Licenses To Remove','remove_licenses')
      );
 
  public function __construct(DbManager $dbManager, UserDao $userDao, LicenseDao $licenseDao = null)
  {
    $this->dbManager = $dbManager;
    $this->userDao = $userDao;
    $this->licenseDao = $licenseDao ?: $GLOBALS['container']->get('dao.license');
  }
 
  public function setDelimiter($delimiter=',')
  {
    $this->delimiter = substr($delimiter,0,1);
  }
 
  public function setEnclosure($enclosure='"')
  {
    $this->enclosure = substr($enclosure,0,1);
  }
 
  public function handleFile($filename, $fileExtension)
  {
    if ($fileExtension === 'json') {
      return $this->handleJsonFile($filename);
    } else {
      return $this->handleCsvFile($filename);
    }
  }
 
  private function handleJsonFile($filename)
  {
    $content = file_get_contents($filename);
    if ($content === false) {
      return _("Could not read JSON file");
    }
 
    $data = json_decode($content, true);
    if (json_last_error() !== JSON_ERROR_NONE) {
      return _("Invalid JSON format: ") . json_last_error_msg();
    }
 
    if (!is_array($data)) {
      return _("JSON file must contain an array of phrases");
    }
 
    return $this->importPhrases($data);
  }
 
  private function handleCsvFile($filename)
  {
    $handle = fopen($filename, 'r');
    if ($handle === false) {
      return _("Could not open CSV file");
    }
 
    $this->headrow = fgetcsv($handle, 0, $this->delimiter, $this->enclosure);
    if ($this->headrow === false) {
      fclose($handle);
      return _("Could not read CSV header");
    }
 
    // Strip BOM from the first header column if present
    $bom = chr(0xEF) . chr(0xBB) . chr(0xBF);
    if (isset($this->headrow[0]) && strpos($this->headrow[0], $bom) === 0) {
      $this->headrow[0] = substr($this->headrow[0], 3);
    }
 
    $data = array();
    $lineNumber = 1;
    while (($row = fgetcsv($handle, 0, $this->delimiter, $this->enclosure)) !== false) {
      $lineNumber++;
      if (count($row) !== count($this->headrow)) {
        fclose($handle);
        return sprintf(_("CSV line %d has %d columns, expected %d"),
                      $lineNumber, count($row), count($this->headrow));
      }
 
      $data[] = array_combine($this->headrow, $row);
    }
    fclose($handle);
 
    return $this->importPhrases($data);
  }
 
  private function importPhrases($data)
  {
    $imported = 0;
    $errors = array();
 
    foreach ($data as $index => $phraseData) {
      try {
        $result = $this->importSinglePhrase($phraseData);
        if ($result['success']) {
          $imported++;
        } else {
          $errors[] = sprintf(_("Row %d: %s"), $index + 1, $result['message']);
        }
      } catch (Exception $e) {
        $errors[] = sprintf(_("Row %d: %s"), $index + 1, $e->getMessage());
      }
    }
 
    $message = sprintf(_("Read file: %d phrases"), $imported);
    if (!empty($errors)) {
      $message .= "\n" . _("Errors:") . "\n" . implode("\n", $errors);
    }
 
    return $message;
  }
 
  private function importSinglePhrase($phraseData)
  {
    // Map headers to standard names
    $mappedData = $this->mapHeaders($phraseData);
 
    // Validate required fields
    if (empty($mappedData['text'])) {
      return array('success' => false, 'message' => _("Text is required"));
    }
 
    // Get current user info
    $userId = Auth::getUserId();
    $groupId = Auth::getGroupId();
 
    // Check for duplicate text
    $textMd5 = md5($mappedData['text']);
    $existingSql = "SELECT cp_pk FROM custom_phrase WHERE text_md5 = $1";
    $existing = $this->dbManager->getSingleRow($existingSql, array($textMd5), __METHOD__ . '.duplicateCheck');
 
    if ($existing) {
      return array('success' => false, 'message' => _("Duplicate text already exists"));
    }
 
    // Insert the phrase
    $insertSql = "INSERT INTO custom_phrase (text, text_md5, acknowledgement, comments, user_fk, group_fk, is_active) 
                  VALUES ($1, $2, $3, $4, $5, $6, $7)";
 
    $params = array(
      $mappedData['text'],
      $textMd5,
      $mappedData['acknowledgement'] ?? '',
      $mappedData['comments'] ?? '',
      $userId,
      $groupId,
      $this->parseBoolean($mappedData['is_active'] ?? false) ? 'true' : 'false'
    );
 
    try {
      $cpPk = $this->dbManager->insertPreparedAndReturn(__METHOD__ . '.insertPhrase', $insertSql, $params, 'cp_pk');
      $message = _("Phrase imported successfully");
 
      $totalAssociated = 0;
      $allFailed = array();
      $allCreated = array();
 
      // Handle licenses to add
      if (!empty($mappedData['licenses_to_add'])) {
        $licenseResult = $this->associateLicenses($cpPk, $mappedData['licenses_to_add'], false);
        $totalAssociated += $licenseResult['associated'];
        $allFailed = array_merge($allFailed, $licenseResult['failed']);
        $allCreated = array_merge($allCreated, $licenseResult['created']);
      }
 
      // Handle licenses to remove
      if (!empty($mappedData['licenses_to_remove'])) {
        $licenseResult = $this->associateLicenses($cpPk, $mappedData['licenses_to_remove'], true);
        $totalAssociated += $licenseResult['associated'];
        $allFailed = array_merge($allFailed, $licenseResult['failed']);
        $allCreated = array_merge($allCreated, $licenseResult['created']);
      }
 
      if (!empty($allCreated)) {
        $message .= ". " . sprintf(_("Created new licenses: %s"), implode(', ', $allCreated));
      }
      if (!empty($allFailed)) {
        $message .= ". " . sprintf(_("Warning: Could not create/find licenses: %s"), implode(', ', $allFailed));
      }
      if ($totalAssociated > 0) {
        $message .= ". " . sprintf(_("Associated %d licenses"), $totalAssociated);
      }
 
      return array('success' => true, 'message' => $message);
    } catch (Exception $e) {
      return array('success' => false, 'message' => _("Failed to import phrase: ") . $e->getMessage());
    }
  }
 
  private function mapHeaders($data)
  {
    $mapped = array();
 
    foreach ($this->alias as $standardName => $aliases) {
      foreach ($aliases as $alias) {
        if (isset($data[$alias])) {
          $mapped[$standardName] = $data[$alias];
          break;
        }
      }
    }
 
    // Normalize array/pipe-separated values from bulk text export format
    $mapped = $this->normalizeBulkExportValues($mapped);
 
    return $mapped;
  }
 
  private function normalizeBulkExportValues($mapped)
  {
    // Join array values to single strings for acknowledgement and comments
    foreach (array('acknowledgement', 'comments') as $field) {
      if (isset($mapped[$field])) {
        if (is_array($mapped[$field])) {
          $mapped[$field] = implode('; ', array_filter($mapped[$field]));
        } elseif (is_string($mapped[$field]) && strpos($mapped[$field], '|') !== false) {
          // Handle pipe-separated values from bulk CSV export
          $parts = array_map('trim', explode('|', $mapped[$field]));
          $mapped[$field] = implode('; ', array_filter($parts));
        }
      }
    }
 
    // Restore literal '\n' escape sequences back to real newlines in text
    if (isset($mapped['text']) && is_string($mapped['text'])) {
      $mapped['text'] = str_replace('\\n', "\n", $mapped['text']);
    }
 
    return $mapped;
  }
 
  private function parseBoolean($value)
  {
    if (is_bool($value)) {
      return $value;
    }
 
    $value = strtolower(trim($value));
    return in_array($value, array('true', '1', 'yes', 'on', 'active'));
  }
 
  private function associateLicenses($cpPk, $licenseNames, $removing = false, $allowCreate = false)
  {
    if (is_array($licenseNames)) {
      $licenseArray = $licenseNames;
    } else {
      // Handle multiple possible separators: comma-space, comma, semicolon, pipe
      $separators = array(', ', ',', ';', '|');
      $licenseArray = array($licenseNames); // Default to single license
 
      foreach ($separators as $separator) {
        if (strpos($licenseNames, $separator) !== false) {
          $licenseArray = array_map('trim', explode($separator, $licenseNames));
          break;
        }
      }
    }
 
    $associatedCount = 0;
    $failedLicenses = array();
    $createdLicenses = array();
 
    foreach ($licenseArray as $licenseName) {
      if (empty($licenseName)) {
        continue;
      }
 
      $normalizedLicenseName = trim($licenseName);
 
      $license = $this->licenseDao->getLicenseByShortName($normalizedLicenseName);
 
      if (!$license) {
        if (!$allowCreate || !$this->isValidLicenseShortname($normalizedLicenseName)) {
          $failedLicenses[] = $licenseName . " (unknown)";
          continue;
        }
 
        try {
          $newLicenseId = $this->licenseDao->insertLicense(
            $normalizedLicenseName,
            '',
            null
          );
          $license = $this->licenseDao->getLicenseById($newLicenseId);
          $createdLicenses[] = $normalizedLicenseName;
        } catch (Exception $e) {
          $failedLicenses[] = $licenseName . " (creation failed)";
          continue;
        }
      }
 
      if ($license) {
        $licenseId = $license->getId();
 
        // Check if association already exists
        $checkSql = "SELECT 1 FROM custom_phrase_license_map WHERE cp_fk = $1 AND rf_fk = $2 LIMIT 1";
        $existing = $this->dbManager->getSingleRow($checkSql, array($cpPk, $licenseId),
                                                  __METHOD__ . '.check.' . $cpPk . '.' . $licenseId);
 
        if (!$existing) {
          // Insert the license association with removing flag
          $insertData = array(
            'cp_fk' => $cpPk,
            'rf_fk' => $licenseId,
            'removing' => $removing ? 'true' : 'false'
          );
 
          try {
            $this->dbManager->insertTableRow('custom_phrase_license_map', $insertData);
            $associatedCount++;
          } catch (Exception $e) {
            $failedLicenses[] = $licenseName . " (insert failed)";
          }
        } else {
          $associatedCount++; // Already exists, count as successful
        }
      }
    }
 
    return array('associated' => $associatedCount, 'failed' => $failedLicenses, 'created' => $createdLicenses);
  }
 
  private function isValidLicenseShortname($shortname)
  {
    // Must not be empty
    if (empty(trim($shortname))) {
      return false;
    }
 
    // Must not exceed 256 characters
    if (strlen($shortname) > 256) {
      return false;
    }
 
    // Must not contain control characters (except spaces)
    if (preg_match('/[\x00-\x1F\x7F]/', $shortname)) {
      return false;
    }
 
    return true;
  }
 
  public function importJsonData($data, string &$msg): string
  {
    $msg = $this->importPhrases($data);
    return $msg;
  }
}