Scheduler operations. More...

#include <libfossrepo.h>
#include <agent.h>
#include <database.h>
#include <event.h>
#include <host.h>
#include <interface.h>
#include <scheduler.h>
#include <fossconfig.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <dirent.h>
#include <fcntl.h>
#include <signal.h>
#include <sys/mman.h>
#include <sys/types.h>
#include <sys/wait.h>
#include <unistd.h>
#include <pwd.h>
#include <grp.h>
#include <glib.h>
#include <gio/gio.h>

Include dependency graph for scheduler.c:

Go to the source code of this file.

Macros
#define	TEST_ERROR(error, ...)

#define	SELECT_DECLS(type, name, l_op, w_op, val) type CONF_##name = val;

#define	MASK_SIGCHLD (1 << 0)

#define	MASK_SIGALRM (1 << 1)

#define	MASK_SIGTERM (1 << 2)

#define	MASK_SIGQUIT (1 << 3)

#define	MASK_SIGHUP (1 << 4)

#define	GU_HEADER "DIRECTORIES"

#define	GU_GROUP "PROJECTGROUP"

#define	GU_USER "PROJECTUSER"

#define	SELECT_CONF_INIT(type, name, l_op, w_op, val)

Functions
void	scheduler_sig_handle (int signo)
	Handles any signals sent to the scheduler that are not SIGCHLD. More...

void	scheduler_signal (scheduler_t *scheduler)
	Function that handles certain signals being delivered to the scheduler. More...

scheduler_t *	scheduler_init (gchar sysconfigdir, log_t log)
	Create a new scheduler object. More...

void	scheduler_destroy (scheduler_t *scheduler)
	Free any memory associated with a scheduler_t. More...

static gboolean	isMaxLimitReached (meta_agent_t *agent)
	Check if the current agent's max limit is respected. More...

void	scheduler_update (scheduler_t *scheduler)
	Update function called after every event. More...

void	set_usr_grp (gchar process_name, fo_conf config)

int	kill_scheduler (int force)
	Kills all other running scheduler. More...

void	scheduler_clear_config (scheduler_t *scheduler)
	Clears any information that is loaded when loading the configuration. More...

static gboolean	g_tree_collect (gpointer key, gpointer value, gpointer data)
	GTraverseFunc used by g_tree_clear to collect all the keys in a tree. More...

void	g_tree_clear (GTree *tree)
	Clears the contents of a GTree. More...

void	scheduler_agent_config (scheduler_t *scheduler)
	Loads a particular agents configuration file. More...

void	scheduler_foss_config (scheduler_t *scheduler)
	Loads the configuration data from fossology.conf. More...

int	scheduler_daemonize (scheduler_t *scheduler)
	Daemonizes the scheduler. More...

void	scheduler_config_event (scheduler_t scheduler, void unused)
	Load both the fossology configuration and all the agent configurations. More...

void	scheduler_close_event (scheduler_t scheduler, void killed)
	Sets the closing flag and possibly kills all currently running agents. More...

void	scheduler_test_agents (scheduler_t scheduler, void unused)
	Event used when the scheduler tests the agents. More...

gint	string_is_num (gchar *str)
	Checks if a string is entirely composed of numeric characters. More...

gint	string_compare (gconstpointer a, gconstpointer b, gpointer user_data)

gint	int_compare (gconstpointer a, gconstpointer b, gpointer user_data)

Variables
int	verbose = 0
	The verbose level. More...

int	closing = 0
	Set if scheduler is shutting down.

GThread *	main_thread
	Pointer to the main thread.

int	sigmask = 0

Detailed Description

Scheduler operations.

Definition in file scheduler.c.

Macro Definition Documentation

◆ SELECT_CONF_INIT

#define SELECT_CONF_INIT	(	type,
		name,
		l_op,
		w_op,
		val
	)

Value:

  if(fo_config_has_key(scheduler->sysconfig, "SCHEDULER", #name))                      \
    CONF_##name = l_op(fo_config_get(scheduler->sysconfig, "SCHEDULER", #name, NULL)); \
  V_SPECIAL("CONFIG: %s == " MK_STRING_LIT(w_op) "\n", #name, CONF_##name );

◆ TEST_ERROR

#define TEST_ERROR	(	error,
		...
	)

Value:

  if(error)                                                        \
  {                                                                \
    log_printf("ERROR %s.%d: %s\n",                                \
      __FILE__, __LINE__, error->message);                         \
    log_printf("ERROR %s.%d: ", __FILE__, __LINE__);               \
    log_printf(__VA_ARGS__);                                       \
    log_printf("\n");                                              \
    g_clear_error(&error);                                         \
    continue;                                                      \
  }

Test if error is not NULL then print it to the log.

Definition at line 44 of file scheduler.c.

Function Documentation

◆ g_tree_clear()

void g_tree_clear ( GTree * tree )

Clears the contents of a GTree.

Parameters

tree	the tree to remove all elements from

Definition at line 721 of file scheduler.c.

◆ g_tree_collect()

static gboolean g_tree_collect	(	gpointer	key,
		gpointer	value,
		gpointer	data
	)

static

GTraverseFunc used by g_tree_clear to collect all the keys in a tree.

Parameters

key	The current key
value	The value mapped to the current key
data	A GList** that the key will be appended to

Returns: Always returns 0

Definition at line 707 of file scheduler.c.

◆ int_compare()

gint int_compare	(	gconstpointer	a,
		gconstpointer	b,
		gpointer	user_data
	)

Utility function that enable the agents to be stored in a GTree using the PID of the associated process.

Parameters

a	The pid of the first process
b	The pid of the second process
user_data	unused in this function

Returns: integral value idicating the relationship between the two pids

Definition at line 1074 of file scheduler.c.

◆ isMaxLimitReached()

static gboolean isMaxLimitReached ( meta_agent_t * agent )

static

Check if the current agent's max limit is respected.

Compare the number of running agents and run limit of the agent.

Parameters

agent Agent which has to be scheduled.

Returns: True if the agent can be scheduled (no. of running agents < max run limit of the agent), false otherwise.

Definition at line 406 of file scheduler.c.

◆ kill_scheduler()

int kill_scheduler ( int force )

Kills all other running scheduler.

Parameters

force if the scheduler should shutdown gracefully

Returns: 0 for success (i.e. a scheduler was killed), -1 for failure.

This uses the /proc file system to find all processes that have fo_scheduler in the name and sends a kill signal to them.

Definition at line 620 of file scheduler.c.

◆ scheduler_agent_config()

void scheduler_agent_config ( scheduler_t * scheduler )

Loads a particular agents configuration file.

This loads and saves the results as a new meta_agent. This assumes that the configuration file for the agent includes the following key/value pairs:

command: The command that will be used to start the agent
max: The maximum number of this agent that can run at once
special: Anything that is special about the agent

Definition at line 743 of file scheduler.c.

◆ scheduler_clear_config()

void scheduler_clear_config ( scheduler_t * scheduler )

Clears any information that is loaded when loading the configuration.

Parameters

scheduler the scheduler to reset the information on

Definition at line 671 of file scheduler.c.

◆ scheduler_close_event()

void scheduler_close_event	(	scheduler_t *	scheduler,
		void *	killed
	)

Sets the closing flag and possibly kills all currently running agents.

This function will cause the scheduler to slowly shutdown. If killed is true this is a quick, ungraceful shutdown.

Parameters

scheduler	the scheduler
killed	should the scheduler kill all currently executing agents before exiting the event loop, or should it wait for them to finished first.

Definition at line 1014 of file scheduler.c.

◆ scheduler_config_event()

void scheduler_config_event	(	scheduler_t *	scheduler,
		void *	unused
	)

Load both the fossology configuration and all the agent configurations.

Parameters

scheduler	the scheduler to load the configuration for
unused	this can be called as an event

Definition at line 991 of file scheduler.c.

◆ scheduler_daemonize()

int scheduler_daemonize ( scheduler_t * scheduler )

Daemonizes the scheduler.

This will make sure that the pid that is maintained in the scheduler struct is correct during the daemonizing process.

Parameters

scheduler the scheduler_t struct

Returns: if the daemonizing was successful.

Definition at line 973 of file scheduler.c.

◆ scheduler_destroy()

void scheduler_destroy ( scheduler_t * scheduler )

Free any memory associated with a scheduler_t.

This will stop the interface if it is currently running, and free all the memory associated with the different regular expression and similar structures.

Parameters

scheduler

Todo:

Interface close

Repo close

Definition at line 362 of file scheduler.c.

◆ scheduler_foss_config()

void scheduler_foss_config ( scheduler_t * scheduler )

Loads the configuration data from fossology.conf.

This assumes that fossology.conf contains the following key/value pairs:

port: the port that the scheduler will listen on
LOG_DIR: the directory that the log should be in

There should be a group named HOSTS with all of the hosts listed as key/value pairs under this category. For each of these hosts, the scheduler will create a new host as an internal representation.

Definition at line 853 of file scheduler.c.

◆ scheduler_init()

scheduler_t* scheduler_init	(	gchar *	sysconfigdir,
		log_t *	log
	)

Create a new scheduler object.

This will initialize everything to a point where it can be used. All regular expressions, GTree's and the job_queue will be correctly created.

Parameters

sysconfigdir	Directory containing the fossology.conf
log	Log file to log messages to

Returns: A new scheduler_t* that can be further populated

Definition at line 249 of file scheduler.c.

◆ scheduler_sig_handle()

void scheduler_sig_handle ( int signo )

Handles any signals sent to the scheduler that are not SIGCHLD.

Currently Handles:

Signal	Effect
SIGCHLD	Scheduler will handle to death of the child process or agent
SIGALRM	Scheduler will run agent updates and database updates
SIGTERM	Scheduler will gracefully shut down
SIGQUIT	Scheduler will forcefully shut down
SIGHIP	Scheduler will reload configuration data

Parameters

signo the number of the signal that was sent

Definition at line 93 of file scheduler.c.

◆ scheduler_signal()

void scheduler_signal ( scheduler_t * scheduler )

Function that handles certain signals being delivered to the scheduler.

This function is called every time the event loop attempts to take something from the event queue. It will also get called once a second regardless of if a new event has been queued.

This function checks the sigmask variable to check what signals have been received since the last time it was called. The sigmask variable should always be accessed atomically since it is accessed by the event loop thread as well as the signal handlers.

Parameters

scheduler Scheduler to sent signal to

Definition at line 142 of file scheduler.c.

◆ scheduler_test_agents()

void scheduler_test_agents	(	scheduler_t *	scheduler,
		void *	unused
	)

Event used when the scheduler tests the agents.

Parameters

scheduler	the scheduler struct
unused

Definition at line 1029 of file scheduler.c.

◆ scheduler_update()

void scheduler_update ( scheduler_t * scheduler )

Update function called after every event.

The heart of the scheduler, the actual scheduling algorithm. This will be passed to the event loop as a call back and will be called every time an event is executed. Therefore the code should be light weight since it will be run very frequently.

Todo:: Currently this will only grab a job and create a single agent to execute the job.

Todo:

Allow for runonpfile jobs to have multiple agents based on size

Allow for job preemption. The scheduler can pause jobs, allow it

Allow for specific hosts to be chosen.

Definition at line 433 of file scheduler.c.

◆ set_usr_grp()

void set_usr_grp	(	gchar *	process_name,
		fo_conf *	config
	)

Correctly set the project user and group. The fossology scheduler must run as the user specified by PROJECT_USER and PROJECT_GROUP since the agents must be able to connect to the database. This ensures that that happens correctly.

Parameters

process_name
config

Definition at line 557 of file scheduler.c.

◆ string_compare()

gint string_compare	(	gconstpointer	a,
		gconstpointer	b,
		gpointer	user_data
	)

Utility function that enables the use of the strcmp function with a GTree.

Parameters

a	The first string
b	The second string
user_data	unused in this function

Returns: Integral value indicating the relationship between the two strings

Definition at line 1060 of file scheduler.c.

◆ string_is_num()

gint string_is_num ( gchar * str )

Checks if a string is entirely composed of numeric characters.

Parameters

str	the string to test

Returns: TRUE if the string is entirely numeric, FALSE otherwise

Definition at line 1041 of file scheduler.c.

Variable Documentation

◆ verbose

int verbose = 0

The verbose level.

The verbose flag for the cli.

Definition at line 57 of file scheduler.c.

Macros

Functions

Variables

Detailed Description

Macro Definition Documentation

◆ SELECT_CONF_INIT

◆ TEST_ERROR

Function Documentation

◆ g_tree_clear()

◆ g_tree_collect()

◆ int_compare()

◆ isMaxLimitReached()

◆ kill_scheduler()

◆ scheduler_agent_config()

◆ scheduler_clear_config()

◆ scheduler_close_event()

◆ scheduler_config_event()

◆ scheduler_daemonize()

◆ scheduler_destroy()

◆ scheduler_foss_config()

◆ scheduler_init()

◆ scheduler_sig_handle()

◆ scheduler_signal()

◆ scheduler_test_agents()

◆ scheduler_update()

◆ set_usr_grp()

◆ string_compare()

◆ string_is_num()

Variable Documentation

◆ verbose