Contains agent related information. More...

#include <agent.h>
#include <database.h>
#include <event.h>
#include <host.h>
#include <job.h>
#include <logging.h>
#include <scheduler.h>
#include <limits.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <time.h>
#include <fcntl.h>
#include <sys/types.h>
#include <sys/wait.h>
#include <unistd.h>
#include <glib.h>

Include dependency graph for agent.c:

Go to the source code of this file.

Classes
struct	agent_spawn_args

Macros
#define	TEST_NULV(a)
	Test if paramater is NULL. More...

#define	TEST_NULL(a, ret)
	Test if paramater is NULL. More...

#define	AGENT_CREDENTIAL

#define	AGENT_LOG_CREDENTIAL

#define	AGENT_ERROR(...)

#define	AGENT_NOTIFY(...)

#define	AGENT_WARNING(...)

#define	AGENT_SEQUENTIAL_PRINT(...)

#define	AGENT_CONCURRENT_PRINT(...)

#define	SELECT_STRING(passed) MK_STRING_LIT(AGENT_##passed),

#define	MAX_CMD_ARGS 30

Functions
static int	agent_close_fd (int pid_ptr, agent_t agent, agent_t *excepted)
	This will close all of the agent's pipes. More...

static int	update (int pid_ptr, agent_t agent, gpointer unused)

static int	agent_kill_traverse (int pid, agent_t agent, gpointer unused)
	GTraversalFunction that kills all of the agents. More...

static int	agent_list (char name, meta_agent_t ma, GOutputStream *ostr)
	GTraverseFunction that will print the name of every agent in alphabetical order separated by spaces. More...

static int	agent_test (const gchar name, meta_agent_t ma, scheduler_t *scheduler)
	GTraversalFunction that tests the current agent on every host. More...

static void	agent_listen (scheduler_t scheduler, agent_t agent)

static void	shell_parse (char confdir, int user_id, int group_id, char input, char jq_cmd_args, int jobId, int argc, char ***argv)
	Parses the shell command that is found in the configuration file. More...

static void *	agent_spawn (agent_spawn_args *pass)
	Spawns a new agent using the command passed in using the meta agent. More...

meta_agent_t *	meta_agent_init (char name, char cmd, int max, int spc)
	Creates a new meta agent. More...

void	meta_agent_destroy (meta_agent_t *ma)

agent_t *	agent_init (scheduler_t scheduler, host_t host, job_t *job)
	Allocate and spawn a new agent. More...

void	agent_destroy (agent_t *agent)
	Frees the memory associated with an agent. More...

void	agent_death_event (scheduler_t scheduler, pid_t pid)

void	agent_create_event (scheduler_t scheduler, agent_t agent)
	Event created when a new agent has been created. More...

void	agent_ready_event (scheduler_t scheduler, agent_t agent)
	Event created when an agent is ready for more data. More...

void	agent_update_event (scheduler_t scheduler, void unused)

void	agent_fail_event (scheduler_t scheduler, agent_t agent)
	Fails an agent. More...

void	list_agents_event (scheduler_t scheduler, GOutputStream ostr)
	Receive agent on interface. More...

void	agent_transition (agent_t *agent, agent_status new_status)

void	agent_pause (agent_t *agent)

void	agent_unpause (agent_t *agent)

void	agent_print_status (agent_t agent, GOutputStream ostr)
	Prints the status of the agent to the output stream provided. More...

void	agent_kill (agent_t *agent)
	Unclean kill of an agent. More...

int	aprintf (agent_t agent, const char fmt,...)

ssize_t	agent_write (agent_t agent, const void buf, int count)

void	test_agents (scheduler_t *scheduler)
	Calls the agent test function for every type of agent. More...

void	kill_agents (scheduler_t *scheduler)
	Call the agent_kill function for every agent within the system. More...

int	add_meta_agent (GTree meta_agents, char name, char *cmd, int max, int spc)

int	is_meta_special (meta_agent_t *ma, int special_type)
	tests if a particular meta agent has a specific special flag set More...

int	is_agent_special (agent_t *agent, int special_type)
	tests if a particular agent has a specific special flag set More...

void	meta_agent_increase_count (meta_agent_t *ma)

void	meta_agent_decrease_count (meta_agent_t *ma)

Variables
const char *	agent_status_strings []

Detailed Description

Contains agent related information.

Todo:: Change the "<date> <time> scheduler ::" to "<date> <time> agent ::" for some log messages

Definition in file agent.c.

Macro Definition Documentation

◆ AGENT_CONCURRENT_PRINT

#define AGENT_CONCURRENT_PRINT ( ... )

Value:

  do {                             \
  AGENT_LOG_CREDENTIAL;                                 \
  con_printf(job_log(agent->owner), __VA_ARGS__); } while(0)

Send logging specifically to the agent log file

Definition at line 97 of file agent.c.

◆ AGENT_CREDENTIAL

#define AGENT_CREDENTIAL

Value:

log_printf("JOB[%d].%s[%d.%s]: ", agent->owner->id, agent->type->name, \

agent->pid, agent->host->name)

Prints the credential of the agent

Definition at line 61 of file agent.c.

◆ AGENT_ERROR

#define AGENT_ERROR ( ... )

Value:

  do {                       \
  log_printf("ERROR: %s.%d: ", __FILE__, __LINE__); \
  AGENT_CREDENTIAL;                                 \
  log_printf(__VA_ARGS__);                          \
  log_printf("\n"); } while(0)

ERROR macro specifically for agents

Definition at line 71 of file agent.c.

◆ AGENT_LOG_CREDENTIAL

#define AGENT_LOG_CREDENTIAL

Value:

con_printf(job_log(agent->owner), "JOB[%d].%s[%d.%s]: ", \

agent->owner->id, agent->type->name, agent->pid, agent->host->name)

Prints the credential to the agent log

Definition at line 66 of file agent.c.

◆ AGENT_NOTIFY

#define AGENT_NOTIFY ( ... )

Value:

  if(TEST_NOTIFY) do { \
  log_printf("NOTE: ");                           \
  AGENT_CREDENTIAL;                            \
  log_printf(__VA_ARGS__);                        \
  log_printf("\n"); } while(0)

NOTIFY macro specifically for agents

Definition at line 78 of file agent.c.

◆ AGENT_SEQUENTIAL_PRINT

#define AGENT_SEQUENTIAL_PRINT ( ... )

Value:

  if(TVERB_AGENT) do { \
  AGENT_CREDENTIAL;                          \
  log_printf(__VA_ARGS__); } while(0)

STANDARD verbose macro changed for agents

Definition at line 92 of file agent.c.

◆ AGENT_WARNING

#define AGENT_WARNING ( ... )

Value:

  if(TEST_WARNING) do {      \
  log_printf("WARNING %s.%d: ", __FILE__, __LINE__);  \
  AGENT_CREDENTIAL;                                   \
  log_printf(__VA_ARGS__);                            \
  log_printf("\n"); } while(0)

WARNING macro specifically for agents

Definition at line 85 of file agent.c.

◆ SELECT_STRING

#define SELECT_STRING ( passed ) MK_STRING_LIT(AGENT_##passed),

Array of C-Strings used to pretty-print the agent status in the log file. Uses the X-Macro defined in agent.h

Definition at line 109 of file agent.c.

◆ TEST_NULL

#define TEST_NULL	(	a,
		ret
	)

Value:

if(!a) { \

errno = EINVAL; ERROR("agent passed is NULL, cannot proceed"); return ret; }

ERROR

#define ERROR(...)

Definition: logging.h:79

Test if paramater is NULL.

Like TEST_NULV(), returns ret instead of void.

Parameters

a	Object to check
ret	Value to return if a is NULL

See also: TEST_NULV()

Definition at line 57 of file agent.c.

◆ TEST_NULV

#define TEST_NULV ( a )

Value:

if(!a) { \

errno = EINVAL; ERROR("agent passed is NULL, cannot proceed"); return; }

Test if paramater is NULL.

If parameter is NULL, set the errno, write the error and return void.

Parameters

a	Object to check

Definition at line 47 of file agent.c.

Function Documentation

◆ add_meta_agent()

int add_meta_agent	(	GTree *	meta_agents,
		char *	name,
		char *	cmd,
		int	max,
		int	spc
	)

Creates a new meta agent and adds it to the list of meta agents. This will parse the shell command that will start the agent process.

Parameters

meta_agents	GTree of meta agents available for the scheduler
name	the name of the meta agent (e.g. "nomos", "copyright", etc...)
cmd	the shell command used to the run the agent
max	the max number of this type of agent that can run concurrently
spc	anything special about the agent type

Definition at line 1316 of file agent.c.

◆ agent_close_fd()

static int agent_close_fd	(	int *	pid_ptr,
		agent_t *	agent,
		agent_t *	excepted
	)

static

This will close all of the agent's pipes.

This function will be called by g_tree_foreach() which is the reason for its formatting.

Parameters

pid_ptr	the key that was used to store this agent
agent	the agent that is being closed
excepted	this is an agent we don't want to close, this is it

Returns: always returns 0 to indicate that the traversal should continue

Definition at line 129 of file agent.c.

◆ agent_create_event()

void agent_create_event	(	scheduler_t *	scheduler,
		agent_t *	agent
	)

Event created when a new agent has been created.

This means that the agent has been allocated internally and the fork() call has successfully executed. The agent has not yet communicated with the scheduler when this event is created.

Parameters

scheduler	the scheduler reference to which agent has to attach
agent	the agent that has been created.

Definition at line 1021 of file agent.c.

◆ agent_death_event()

void agent_death_event	(	scheduler_t *	scheduler,
		pid_t *	pid
	)

Event created when a SIGCHLD is received for an agent. If one SIGCHILD is received for several process deaths, there will be seperate events for each pid.

Parameters

scheduler	the scheduler reference to which process was attached to
pid	the pid of the process that died

Definition at line 952 of file agent.c.

◆ agent_destroy()

void agent_destroy ( agent_t * agent )

Frees the memory associated with an agent.

This include:

All of the files that are open in the agent
All of the pipes still open for the agent
Inform the os that the process can die using a waitpid()
Free the internal data structure of the agent

Parameters

agent the agent to destroy

Definition at line 924 of file agent.c.

◆ agent_fail_event()

void agent_fail_event	(	scheduler_t *	scheduler,
		agent_t *	agent
	)

Fails an agent.

This will move the agent status to AG_FAILED and send a SIGKILL to the relevant agent. It will also update the agents status within the job that owns it and close the associated communication thread.

Parameters

scheduler	the scheduler to which agent is attached
agent	the agent that is failing.

Definition at line 1102 of file agent.c.

◆ agent_init()

agent_t* agent_init	(	scheduler_t *	scheduler,
		host_t *	host,
		job_t *	job
	)

Allocate and spawn a new agent.

The agent that is spawned will be of the same type as the meta_agent that is passed to this function and the agent will run on the host that is passed.

Parameters

scheduler	the scheduler this agent is being created under
host	the machine to start the agent on
job	the job that this agent belongs to

Definition at line 802 of file agent.c.

◆ agent_kill()

void agent_kill ( agent_t * agent )

Unclean kill of an agent.

This simply sends a SIGKILL to the agent and lets everything else get cleaned up normally.

Parameters

agent the agent to kill

Definition at line 1223 of file agent.c.

◆ agent_kill_traverse()

static int agent_kill_traverse	(	int *	pid,
		agent_t *	agent,
		gpointer	unused
	)

static

GTraversalFunction that kills all of the agents.

This is used for an unclean death since all of the child processes will be sent a kill signal instead of existing cleanly.

Parameters

pid	the process id associated with the agent
agent	a pointer to the information associated with an agent
unused

Returns: always returns 0 to indicate that the traversal should continue

Definition at line 202 of file agent.c.

◆ agent_list()

static int agent_list	(	char *	name,
		meta_agent_t *	ma,
		GOutputStream *	ostr
	)

static

GTraverseFunction that will print the name of every agent in alphabetical order separated by spaces.

Parameters

name	the name of the agent
ma	the meta_agents structure associated with the specific name
ostr	the output stream to write the data to, socket in this case

Returns: always returns 0 to indicate that the traversal should continue

Definition at line 217 of file agent.c.

◆ agent_listen()

static void agent_listen	(	scheduler_t *	scheduler,
		agent_t *	agent
	)

static

Main function used for agent communication. This is where the communication thread will spend the majority of its time.

Parameters

scheduler	Pointer to scheduler interface
agent	a the agent that will be listened on

Start by getting the version information from the agent. The agent should send "VERSION: <string>" where the string is the version information. there are five things that can happen here.

the agent sends correct version information => continue
this is the first agent to send version info => save version and continue
the agent sends incorrect version information => invalidate the agent
the agent doesn't send version information => invalidate the agent
the agent crashed before sending information => close the thread

If we reach here the agent has correctly sent VERION information to the scheduler. The agent now enters a listening loop. The communication thread will wait for input from the agent, and act according to the agents current state and what was sent

Note: any command prepended by "@@@" is a message from the scheduler to the communication thread, not from the agent.

command: "BYE"

The agent has finished processing all of the data from the relevant job. This command is follow by a return code. 0 indicates that it completed correctly, anything else can be used as an error code. Regardless of whether the agent completed, the communication thread will shutdown.
command "@@@1"

The scheduler needs the communication thread to shutdown. This will normally only happen if the agent crashes and the scheduler receives a SIGCHLD for it before it sends "BYE #".
command "@@@0"

The scheduler has updated the data that the agent should be processing. This is sent after an agent sends the "OK" command, and the scheduler has processed the resulting agent_ready_event().
command: "OK"

The agent is ready for data. This is sent it 2 situations:
1. the agent has completed startup and is ready for the first part of the data that needs to be analyzed for the job
2. the agent has finished the last piece of the job it was working on and is ready for the next piece or to be shutdown
command: "HEART"

Given the size of jobs that can be processed by FOSSology, agents can take an extremely long period of time to finish. To make sure that an agent is still working it must periodically update the scheduler with how much of the job it has processed.
command: "EMAIL"

Agents have the ability to set the message that will be sent with the notification email. This grabs the message and sets inside the job that the agent is running under.
command: "SPECIAL"

Agents can set special attributes that change how it is treated during execution. This grabs the command and whether it is being set to true or false. Agents use this by calling fo_scheduler_set_special() in the agent api.
command: GETSPECIAL

The agent has requested the value of a special attribute. The scheduler will respond with the value of the special attribute.
command: unknown

The agent didn't use a legal command. This will simply put what the agent printed into the log and move on.

Definition at line 264 of file agent.c.

◆ agent_pause()

void agent_pause ( agent_t * agent )

Pauses an agent, this will pause the agent by sending a SIGSTOP to the process and then decrease the load on the host machine.

Parameters

agent the agent to pause

Definition at line 1164 of file agent.c.

◆ agent_print_status()

void agent_print_status	(	agent_t *	agent,
		GOutputStream *	ostr
	)

Prints the status of the agent to the output stream provided.

The formating for this is as such:

`agent:<pid> host:<host> type:<type> status:<status> time:<time>`

Parameters

agent	Agent to get info from
ostr	Stream to write info to

Definition at line 1193 of file agent.c.

◆ agent_ready_event()

void agent_ready_event	(	scheduler_t *	scheduler,
		agent_t *	agent
	)

Event created when an agent is ready for more data.

This will event will be created when an agent first communicates with the scheduler, so this will handle changing its status to AG_RUNNING. This will also be created every time an agent finishes a block of data.

Parameters

scheduler	the scheduler reference to which agent is attached
agent	the agent that is ready

Definition at line 1042 of file agent.c.

◆ agent_spawn()

static void* agent_spawn ( agent_spawn_args * pass )

static

Spawns a new agent using the command passed in using the meta agent.

This function will call the fork and exec necessary to create a new agent. As a result what this function does will change depending on if it is running in the child or the parent.

child: Will duplicate the stdin, stdout, and stderr pipes for printing to the scheduler, parse the command line options for the agent and start the agent. It will then call exec to start the new agent process

parent: This will enter the listen function, and wait for information from the child, either as a failure or as an update for the information being analyzed

Parameters

pass	a pointer to scheduler_t and the new agent_t

Definition at line 629 of file agent.c.

◆ agent_test()

static int agent_test	(	const gchar *	name,
		meta_agent_t *	ma,
		scheduler_t *	scheduler
	)

static

GTraversalFunction that tests the current agent on every host.

This will traverse the list of hosts and start an agent that is of the type of the current meta agent on every host.

Parameters

name	The name of the meta agent
ma	The meta_agent structure needed for agent creation
scheduler	The scheduler object to test the agents on

Returns: Always return 0

Definition at line 238 of file agent.c.

◆ agent_transition()

void agent_transition	(	agent_t *	agent,
		agent_status	new_status
	)

Changes the status of the agent internal to the scheduler. This function is used to transition between agent states instead of a raw set of the status so that correct printing of the verbose message is guaranteed

Parameters

agent	the agent to change the status for
new_status	the new status of the agentchar* sysconfdir = NULL; // system configuration directory (SYSCONFDIR)

Definition at line 1136 of file agent.c.

◆ agent_unpause()

void agent_unpause ( agent_t * agent )

Unpause the agent, this will send a SIGCONT to the process regardless of if a SIGCONT was sent. If the process wasn't SIGSTOP'd this will do nothing. Also increases the load on the host.

Parameters

agent the agent to unpause

Definition at line 1177 of file agent.c.

◆ agent_update_event()

void agent_update_event	(	scheduler_t *	scheduler,
		void *	unused
	)

Event created when the scheduler receives a SIGALRM. This will loop over every agent and call the update function on it. This will kill any agents that are hung without heart beat or any agents that have stopped updating the number of item processed.

Parameters

scheduler	the scheduler reference to inform
unused	needed since this an event, but should be NULL

Definition at line 1087 of file agent.c.

◆ agent_write()

ssize_t agent_write	(	agent_t *	agent,
		const void *	buf,
		int	count
	)

Write information to the communication thread for the agent. This is used when the scheduler needs to wake up or kill the thread used to talk to the agent. When using this function, one should always print "@@@..." where ... is the message that is actually getting sent.

Parameters

agent	the agent to send the information to
buf	the actual data
count	the number of bytes to write to the agent

Returns: returns if the write was successful

Definition at line 1274 of file agent.c.

◆ aprintf()

int aprintf	(	agent_t *	agent,
		const char *	fmt,
			...
	)

Acts as a standard printf, but prints the agents instead of stdout. This is the main function used by the scheduler when communicating with the agents.

Parameters

agent	the agent to send the formated data to
fmt	the formating string for the data

Returns: if the print was successful

Definition at line 1238 of file agent.c.

◆ is_agent_special()

int is_agent_special	(	agent_t *	agent,
		int	special_type
	)

tests if a particular agent has a specific special flag set

Parameters

a	the agent that should be tested
special_type	in what way is the agent special

Returns: true or false

Definition at line 1353 of file agent.c.

◆ is_meta_special()

int is_meta_special	(	meta_agent_t *	ma,
		int	special_type
	)

tests if a particular meta agent has a specific special flag set

Parameters

ma	the meta agent that should be checked
special_type	in what way is the agent special

Returns: true or false

Definition at line 1341 of file agent.c.

◆ kill_agents()

void kill_agents ( scheduler_t * scheduler )

Call the agent_kill function for every agent within the system.

This will send a SIGKILL to every child process of the scheduler. Used when shutting down the scheduler.

Definition at line 1301 of file agent.c.

◆ list_agents_event()

void list_agents_event	(	scheduler_t *	scheduler,
		GOutputStream *	ostr
	)

Receive agent on interface.

Calls agent_list() and print evert agent attached to the scheduler.

Parameters

scheduler	the scheduler to which agent is attached
ostr	Stream to write info to

Definition at line 1118 of file agent.c.

◆ meta_agent_decrease_count()

void meta_agent_decrease_count ( meta_agent_t * ma )

Decrease the running agent count.

Parameters

ma	Agent's meta

Definition at line 1372 of file agent.c.

◆ meta_agent_destroy()

void meta_agent_destroy ( meta_agent_t * ma )

Free the memory associated with a meta_agent. This is a destructor, and as a result the meta_agent should not be used again after a call to this method

Parameters

ma	the meta_agent to clear

Definition at line 784 of file agent.c.

◆ meta_agent_increase_count()

void meta_agent_increase_count ( meta_agent_t * ma )

Increase the running agent count.

Parameters

ma	Agent's meta

Definition at line 1362 of file agent.c.

◆ meta_agent_init()

meta_agent_t* meta_agent_init	(	char *	name,
		char *	cmd,
		int	max,
		int	spc
	)

Creates a new meta agent.

This will take and parse the information necessary for the creation of a new agent instance. The name of the agent, the cmd for starting the agent, the number of these agents that can run simutaniously, and any special conditions for this agent. This function is where the cmd will get parsed to be passed as command line args to the new agent.

Parameters

name	the name of the agent (i.e. nomos, buckets, etc...)
cmd	the command for starting the agent in a shell
max	the number of these that can concurrently, -1 for no limit
spc	any special conditions associated with the agent

Returns: meta_agent_t Meta agent

Definition at line 744 of file agent.c.

◆ shell_parse()

static void shell_parse	(	char *	confdir,
		int	user_id,
		int	group_id,
		char *	input,
		char *	jq_cmd_args,
		int	jobId,
		int *	argc,
		char ***	argv
	)

static

Parses the shell command that is found in the configuration file.

Parameters

	confdir	The configuration directory for FOSSology
	user_id	The id of the user that created the job
	group_id	The id of the group that created the job
	input	The command line that was in the agent configuration file
	jq_cmd_args	Extra parameters required by agent (if any)
	jobId	Job id (from db)
[out]	argc	Returns the number of arguments parsed
[out]	argv	The parsed arguments

Definition at line 551 of file agent.c.

◆ test_agents()

void test_agents ( scheduler_t * scheduler )

Calls the agent test function for every type of agent.

This is used when either the -t or -T option are used upon scheduler creation.

Parameters

scheduler scheduler reference to test attached agents

Definition at line 1290 of file agent.c.

◆ update()

static int update	(	int *	pid_ptr,
		agent_t *	agent,
		gpointer	unused
	)

static

Check the status and check in time of an agent.

If we haven't gotten a recent communication, close it
If it hasn't been performing tasks, close it

Parameters

pid_ptr	pointer to key in g_tree, is not used in this function
agent	the agent that needs to be updated
unused	data that is also not used in this function

Returns: always returns 0 to indicate that the traversal should continue

Definition at line 152 of file agent.c.

Variable Documentation

◆ agent_status_strings

const char* agent_status_strings[]

Initial value:

=

{ AGENT_STATUS_TYPES(SELECT_STRING) }

SELECT_STRING

#define SELECT_STRING(passed)

Definition: agent.c:109

AGENT_STATUS_TYPES

#define AGENT_STATUS_TYPES(apply)

Definition: agent.h:50

Definition at line 110 of file agent.c.

Classes

Macros

Functions

Variables

Detailed Description

Macro Definition Documentation

◆ AGENT_CONCURRENT_PRINT

◆ AGENT_CREDENTIAL

◆ AGENT_ERROR

◆ AGENT_LOG_CREDENTIAL

◆ AGENT_NOTIFY

◆ AGENT_SEQUENTIAL_PRINT

◆ AGENT_WARNING

◆ SELECT_STRING

◆ TEST_NULL

◆ TEST_NULV

Function Documentation

◆ add_meta_agent()

◆ agent_close_fd()

◆ agent_create_event()

◆ agent_death_event()

◆ agent_destroy()

◆ agent_fail_event()

◆ agent_init()

◆ agent_kill()

◆ agent_kill_traverse()

◆ agent_list()

◆ agent_listen()

◆ agent_pause()

◆ agent_print_status()

◆ agent_ready_event()

◆ agent_spawn()

◆ agent_test()

◆ agent_transition()

◆ agent_unpause()

◆ agent_update_event()

◆ agent_write()

◆ aprintf()

◆ is_agent_special()

◆ is_meta_special()

◆ kill_agents()

◆ list_agents_event()

◆ meta_agent_decrease_count()

◆ meta_agent_destroy()

◆ meta_agent_increase_count()

◆ meta_agent_init()

◆ shell_parse()

◆ test_agents()

◆ update()

Variable Documentation

◆ agent_status_strings