changeset 24:34cbbd215ef7

misc: add basic support for jobresults
author David Demelier <markand@malikania.fr>
date Mon, 25 Jul 2022 21:11:23 +0200
parents 2cb228f23f53
children c40f98360ac9
files .hgignore Makefile config.mk lib/apic.c lib/db.c lib/log.c scictl/scictl.c scid/http.c scid/page-api-jobresults.c scid/page-api-jobresults.h sciworkerd/sciworkerd.c sciworkerd/task.c
diffstat 12 files changed, 148 insertions(+), 41 deletions(-) [+]
line wrap: on
line diff
--- a/.hgignore	Thu Jul 21 21:55:02 2022 +0200
+++ b/.hgignore	Mon Jul 25 21:11:23 2022 +0200
@@ -5,13 +5,11 @@
 \.o$
 ^sql/.*\.h$
 
-# generated manpages.
-^man/.*\.[1-9]$
-
 # vim/emacs specific.
 ^tags$
-^tags.lock$
-^tags.temp$
+^tags\.lock$
+^tags\.temp$
+^cscope\.out$
 \.swp$
 \.swo$
 
--- a/Makefile	Thu Jul 21 21:55:02 2022 +0200
+++ b/Makefile	Mon Jul 25 21:11:23 2022 +0200
@@ -52,6 +52,7 @@
 SCID=                   scid/scid
 SCID_SRCS=              scid/http.c                     \
                         scid/main.c                     \
+                        scid/page-api-jobresults.c      \
                         scid/page-api-jobs.c            \
                         scid/page-api-projects.c        \
                         scid/page-api-todo.c            \
--- a/config.mk	Thu Jul 21 21:55:02 2022 +0200
+++ b/config.mk	Mon Jul 25 21:11:23 2022 +0200
@@ -1,5 +1,5 @@
 CC=             cc
-CFLAGS=         -g -O0
+CFLAGS=         -g -O0 -Wall -Wextra
 #CFLAGS=         -Wall -Wextra -fsanitize=address,undefined -g -O0
 #LDFLAGS=        -fsanitize=address,undefined
 
--- a/lib/apic.c	Thu Jul 21 21:55:02 2022 +0200
+++ b/lib/apic.c	Mon Jul 25 21:11:23 2022 +0200
@@ -177,7 +177,7 @@
 	json_decref(doc);
 	free(body);
 
-	return 0;
+	return ret;
 }
 
 static json_t *
@@ -290,7 +290,7 @@
 
 	struct converter cv = {
 		.data = jobs,
-		.datasz = 1,
+		.datasz = jobsz,
 		.unpack = wrap_job_from
 	};
 
--- a/lib/db.c	Thu Jul 21 21:55:02 2022 +0200
+++ b/lib/db.c	Mon Jul 25 21:11:23 2022 +0200
@@ -131,6 +131,8 @@
 	return ret;
 }
 
+#if 0
+
 static int
 update(const char *sql, const char *fmt, ...)
 {
@@ -158,6 +160,8 @@
 	return ret;
 }
 
+#endif
+
 static ssize_t
 list(struct list *sel, const char *sql, const char *args, ...)
 {
--- a/lib/log.c	Thu Jul 21 21:55:02 2022 +0200
+++ b/lib/log.c	Mon Jul 25 21:11:23 2022 +0200
@@ -66,6 +66,7 @@
 
 	vsnprintf(line, sizeof (line), fmt, ap);
 	syslog(syslog_levels[level], "%s", line);
+	printf("%s\n", line);
 }
 
 void
--- a/scictl/scictl.c	Thu Jul 21 21:55:02 2022 +0200
+++ b/scictl/scictl.c	Mon Jul 25 21:11:23 2022 +0200
@@ -98,12 +98,6 @@
 	return console;
 }
 
-static size_t
-extract(char *s, size_t w, size_t n, void *data)
-{
-	return fwrite(s, w, n, data);
-}
-
 static void
 cmd_job_add(int argc, char **argv)
 {
@@ -254,7 +248,7 @@
 	if ((projectsz = apic_project_list(&req, projects, UTIL_SIZE(projects))) < 0)
 		util_die("abort: unable to list projects: %s\n", req.error);
 
-	for (size_t i = 0; i < projectsz; ++i) {
+	for (ssize_t i = 0; i < projectsz; ++i) {
 		printf("%-16s%s\n", "name:", projects[i].name);
 		printf("%-16s%s\n", "desc:", projects[i].desc);
 		printf("%-16s%s\n", "url:", projects[i].url);
@@ -300,7 +294,7 @@
 	if ((wksz = apic_worker_list(&req, wk, UTIL_SIZE(wk))) < 0)
 		util_die("abort: unable to list worker: %s\n", req.error);
 
-	for (size_t i = 0; i < wksz; ++i) {
+	for (ssize_t i = 0; i < wksz; ++i) {
 		printf("%-16s%s\n", "name:", wk[i].name);
 		printf("%-16s%s\n", "desc:", wk[i].desc);
 
--- a/scid/http.c	Thu Jul 21 21:55:02 2022 +0200
+++ b/scid/http.c	Mon Jul 25 21:11:23 2022 +0200
@@ -27,6 +27,7 @@
 
 #include "http.h"
 #include "log.h"
+#include "page-api-jobresults.h"
 #include "page-api-jobs.h"
 #include "page-api-projects.h"
 #include "page-api-todo.h"
@@ -45,6 +46,7 @@
 		const char *prefix;
 		void (*handler)(struct kreq *);
 	} apis[] = {
+		{ "v1/jobresults",      page_api_v1_jobresults  },
 		{ "v1/jobs",            page_api_v1_jobs        },
 		{ "v1/projects",        page_api_v1_projects    },
 		{ "v1/todo",            page_api_v1_todo        },
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/scid/page-api-jobresults.c	Mon Jul 25 21:11:23 2022 +0200
@@ -0,0 +1,82 @@
+/*
+ * page-api-jobresults.c -- /api/v?/jobresults route
+ *
+ * Copyright (c) 2021 David Demelier <markand@malikania.fr>
+ *
+ * Permission to use, copy, modify, and/or distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#include <sys/types.h>
+#include <assert.h>
+#include <stdarg.h>
+#include <stdint.h>
+#include <kcgi.h>
+
+#include "db.h"
+#include "log.h"
+#include "page.h"
+#include "types.h"
+
+static int
+save(const char *json)
+{
+	struct jobresult res = {0};
+	int ret = -1;
+
+	json_t *doc;
+	json_error_t err;
+
+	if (!(doc = json_loads(json, 0, &err)))
+		log_warn("api/post: invalid JSON input: %s", err.text);
+	else if (jobresult_from(&res, 1, doc) < 0)
+		log_warn("api/post: failed to decode parameters");
+	else if (db_jobresult_add(&res) < 0)
+		log_warn("api/post: database save error");
+	else
+		ret = 0;
+
+	json_decref(doc);
+	jobresult_finish(&res);
+
+	return ret;
+}
+
+static void
+post(struct kreq *r)
+{
+	if (r->fieldsz < 1)
+		page(r, NULL, KHTTP_400, KMIME_APP_JSON, NULL);
+	else if (save(r->fields[0].val) < 0)
+		page(r, NULL, KHTTP_500, KMIME_APP_JSON, NULL);
+	else {
+		khttp_head(r, kresps[KRESP_CONTENT_TYPE], "%s", kmimetypes[KMIME_APP_JSON]);
+		khttp_head(r, kresps[KRESP_STATUS], "%s", khttps[KHTTP_200]);
+		khttp_body(r);
+		khttp_free(r);
+	}
+}
+
+void
+page_api_v1_jobresults(struct kreq *r)
+{
+	assert(r);
+
+	switch (r->method) {
+	case KMETHOD_POST:
+		post(r);
+		break;
+	default:
+		page(r, NULL, KHTTP_400, KMIME_APP_JSON, NULL);
+		break;
+	}
+}
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/scid/page-api-jobresults.h	Mon Jul 25 21:11:23 2022 +0200
@@ -0,0 +1,27 @@
+/*
+ * page-api-jobresults.h -- /api/v?/jobresults route
+ *
+ * Copyright (c) 2021 David Demelier <markand@malikania.fr>
+ *
+ * Permission to use, copy, modify, and/or distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#ifndef SCI_PAGE_API_JOBRESULTS_H
+#define SCI_PAGE_API_JOBRESULTS_H
+
+struct kreq;
+
+void
+page_api_v1_jobresults(struct kreq *);
+
+#endif /* !SCI_PAGE_API_JOBRESULTS_H */
--- a/sciworkerd/sciworkerd.c	Thu Jul 21 21:55:02 2022 +0200
+++ b/sciworkerd/sciworkerd.c	Mon Jul 25 21:11:23 2022 +0200
@@ -13,7 +13,7 @@
 #include "types.h"
 #include "util.h"
 
-#define TAG "sigworkerd: "
+#define TAG "sciworkerd: "
 
 struct taskentry {
 	struct task *task;
@@ -48,7 +48,7 @@
 static void
 stop(int sign)
 {
-	log_info(TAG "exiting on signal %d\n", sign);
+	log_info(TAG "exiting on signal %d", sign);
 	run = 0;
 }
 
@@ -69,7 +69,7 @@
 {
 	struct taskentry *tk;
 
-	log_info(TAG "queued job build (%d) for tag %s\n", job->id, job->tag);
+	log_info(TAG "queued job build (%d) for tag %s", job->id, job->tag);
 
 	tk = util_calloc(1, sizeof (*tk));
 	tk->task = task_new(job->tag);
@@ -82,7 +82,7 @@
 {
 	size_t total = 0;
 
-	for (ssize_t i = 0; i < jobsz; ++i) {
+	for (size_t i = 0; i < jobsz; ++i) {
 		if (!pending(jobs[i].id)) {
 			queue(&jobs[i]);
 			total++;
@@ -130,9 +130,7 @@
 	if (apic_worker_find(&req, &worker, sciworkerd.name) < 0)
 		log_die(TAG "unable to fetch worker info: %s", req.error);
 
-	log_info("worker name: %s", worker.name);
-	log_info("worker description: %s", worker.desc);
-
+	log_info("sciworkerd: worker %s (%s)", worker.name, worker.desc);
 	apic_finish(&req);
 }
 
@@ -191,10 +189,11 @@
 	size_t running = count(tasks);
 	struct taskentry *entry;
 
-	while (running-- > 0 && (entry = taskpending)) {
+	while (running < sciworkerd.maxjobs && (entry = taskpending)) {
 		if (start(entry) < 0)
 			delete(entry);
 		else {
+			running++;
 			LL_DELETE(taskpending, entry);
 			LL_APPEND(tasks, entry);
 		}
@@ -208,6 +207,7 @@
 	struct taskcode code;
 	struct pollfd *fds;
 	size_t fdsz, i = 0;
+	pid_t pid;
 	int ret;
 
 	/* First, read every pipes. */
@@ -239,6 +239,8 @@
 
 		/* Now wait for the task to complete. */
 		if (ret <= 0) {
+			pid = task_pid(iter->task);
+
 			if (task_wait(iter->task) < 0)
 				log_warn(TAG "task wait error: %s", strerror(errno));
 			else {
@@ -247,11 +249,11 @@
 				switch (task_status(iter->task)) {
 				case TASKSTATUS_EXITED:
 					log_info(TAG "task %lld exited with code %d",
-					    (long long int)task_pid(iter->task), code.exitcode);
+					    (long long int)pid, code.exitcode);
 					break;
 				case TASKSTATUS_KILLED:
 					log_info(TAG "task %lld killed with signal %d",
-					    (long long int)task_pid(iter->task), code.sigcode);
+					    (long long int)pid, code.sigcode);
 					break;
 				default:
 					break;
@@ -259,10 +261,11 @@
 			}
 
 			/* Remove that task and push to the outgoing queue. */
-			next = iter->next;
 			LL_DELETE(tasks, iter);
 			LL_APPEND(taskfinished, iter);
 		}
+
+		iter = next;
 	}
 
 	free(fds);
@@ -275,7 +278,6 @@
 ghost_all(void)
 {
 	struct taskentry *iter, *tmp;
-	time_t now;
 
 	LL_FOREACH_SAFE(tasks, iter, tmp) {
 		if (difftime(time(NULL), task_uptime(iter->task)) < sciworkerd.timeout)
@@ -304,20 +306,15 @@
 		.worker_name = worker.name
 	};
 	struct apic req;
-	json_t *doc;
-	int ret; 
-
-	doc = jobresult_to(&res, 1);
-	ret = apic_post(&req, doc, "%s/api/v1/jobs", sciworkerd.url);
-	json_decref(doc);
+	int ret = 0; 
 
-	if (ret)
+	if (apic_jobresult_add(&req, &res) < 0) {
 		log_warn(TAG "unable to publish task: %s", req.error);
-	else
-		log_info(TAG "task successfully published");
+		ret = -1;
+	}
 
+	apic_finish(&req);
 	jobresult_finish(&res);
-	apic_finish(&req);
 
 	return ret;
 }
@@ -358,6 +355,7 @@
 {
 	while (run) {
 		fetch_jobs();
+		start_all();
 		process_all();
 		ghost_all();
 		publish_all();
--- a/sciworkerd/task.c	Thu Jul 21 21:55:02 2022 +0200
+++ b/sciworkerd/task.c	Mon Jul 25 21:11:23 2022 +0200
@@ -65,7 +65,7 @@
 		goto failed;
 	if (fchmod(self->scriptfd, MODE) < 0)
 		goto failed;
-	if (write(self->scriptfd, script, len) != len)
+	if (write(self->scriptfd, script, len) != (ssize_t)len)
 		goto failed;
 
 	return 0;
@@ -107,7 +107,7 @@
 		break;
 	}
 
-	return 0;
+	return self->child;
 }
 
 int
@@ -205,7 +205,7 @@
 	if (fd->revents & POLLIN) {
 		if ((nr = read(self->pipe[0], buf, sizeof (buf))) <= 0)
 			return nr;
-		if (fwrite(buf, 1, nr, self->fp) != nr)
+		if (fwrite(buf, 1, nr, self->fp) != (size_t)nr)
 			return -1;
 	}