# HG changeset patch # User David Demelier # Date 1658776283 -7200 # Node ID 34cbbd215ef761ac7d9ed3fa09e82f4f6cf6f374 # Parent 2cb228f23f53077a8c34005941f488619c3053b3 misc: add basic support for jobresults diff -r 2cb228f23f53 -r 34cbbd215ef7 .hgignore --- a/.hgignore Thu Jul 21 21:55:02 2022 +0200 +++ b/.hgignore Mon Jul 25 21:11:23 2022 +0200 @@ -5,13 +5,11 @@ \.o$ ^sql/.*\.h$ -# generated manpages. -^man/.*\.[1-9]$ - # vim/emacs specific. ^tags$ -^tags.lock$ -^tags.temp$ +^tags\.lock$ +^tags\.temp$ +^cscope\.out$ \.swp$ \.swo$ diff -r 2cb228f23f53 -r 34cbbd215ef7 Makefile --- a/Makefile Thu Jul 21 21:55:02 2022 +0200 +++ b/Makefile Mon Jul 25 21:11:23 2022 +0200 @@ -52,6 +52,7 @@ SCID= scid/scid SCID_SRCS= scid/http.c \ scid/main.c \ + scid/page-api-jobresults.c \ scid/page-api-jobs.c \ scid/page-api-projects.c \ scid/page-api-todo.c \ diff -r 2cb228f23f53 -r 34cbbd215ef7 config.mk --- a/config.mk Thu Jul 21 21:55:02 2022 +0200 +++ b/config.mk Mon Jul 25 21:11:23 2022 +0200 @@ -1,5 +1,5 @@ CC= cc -CFLAGS= -g -O0 +CFLAGS= -g -O0 -Wall -Wextra #CFLAGS= -Wall -Wextra -fsanitize=address,undefined -g -O0 #LDFLAGS= -fsanitize=address,undefined diff -r 2cb228f23f53 -r 34cbbd215ef7 lib/apic.c --- a/lib/apic.c Thu Jul 21 21:55:02 2022 +0200 +++ b/lib/apic.c Mon Jul 25 21:11:23 2022 +0200 @@ -177,7 +177,7 @@ json_decref(doc); free(body); - return 0; + return ret; } static json_t * @@ -290,7 +290,7 @@ struct converter cv = { .data = jobs, - .datasz = 1, + .datasz = jobsz, .unpack = wrap_job_from }; diff -r 2cb228f23f53 -r 34cbbd215ef7 lib/db.c --- a/lib/db.c Thu Jul 21 21:55:02 2022 +0200 +++ b/lib/db.c Mon Jul 25 21:11:23 2022 +0200 @@ -131,6 +131,8 @@ return ret; } +#if 0 + static int update(const char *sql, const char *fmt, ...) { @@ -158,6 +160,8 @@ return ret; } +#endif + static ssize_t list(struct list *sel, const char *sql, const char *args, ...) { diff -r 2cb228f23f53 -r 34cbbd215ef7 lib/log.c --- a/lib/log.c Thu Jul 21 21:55:02 2022 +0200 +++ b/lib/log.c Mon Jul 25 21:11:23 2022 +0200 @@ -66,6 +66,7 @@ vsnprintf(line, sizeof (line), fmt, ap); syslog(syslog_levels[level], "%s", line); + printf("%s\n", line); } void diff -r 2cb228f23f53 -r 34cbbd215ef7 scictl/scictl.c --- a/scictl/scictl.c Thu Jul 21 21:55:02 2022 +0200 +++ b/scictl/scictl.c Mon Jul 25 21:11:23 2022 +0200 @@ -98,12 +98,6 @@ return console; } -static size_t -extract(char *s, size_t w, size_t n, void *data) -{ - return fwrite(s, w, n, data); -} - static void cmd_job_add(int argc, char **argv) { @@ -254,7 +248,7 @@ if ((projectsz = apic_project_list(&req, projects, UTIL_SIZE(projects))) < 0) util_die("abort: unable to list projects: %s\n", req.error); - for (size_t i = 0; i < projectsz; ++i) { + for (ssize_t i = 0; i < projectsz; ++i) { printf("%-16s%s\n", "name:", projects[i].name); printf("%-16s%s\n", "desc:", projects[i].desc); printf("%-16s%s\n", "url:", projects[i].url); @@ -300,7 +294,7 @@ if ((wksz = apic_worker_list(&req, wk, UTIL_SIZE(wk))) < 0) util_die("abort: unable to list worker: %s\n", req.error); - for (size_t i = 0; i < wksz; ++i) { + for (ssize_t i = 0; i < wksz; ++i) { printf("%-16s%s\n", "name:", wk[i].name); printf("%-16s%s\n", "desc:", wk[i].desc); diff -r 2cb228f23f53 -r 34cbbd215ef7 scid/http.c --- a/scid/http.c Thu Jul 21 21:55:02 2022 +0200 +++ b/scid/http.c Mon Jul 25 21:11:23 2022 +0200 @@ -27,6 +27,7 @@ #include "http.h" #include "log.h" +#include "page-api-jobresults.h" #include "page-api-jobs.h" #include "page-api-projects.h" #include "page-api-todo.h" @@ -45,6 +46,7 @@ const char *prefix; void (*handler)(struct kreq *); } apis[] = { + { "v1/jobresults", page_api_v1_jobresults }, { "v1/jobs", page_api_v1_jobs }, { "v1/projects", page_api_v1_projects }, { "v1/todo", page_api_v1_todo }, diff -r 2cb228f23f53 -r 34cbbd215ef7 scid/page-api-jobresults.c --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/scid/page-api-jobresults.c Mon Jul 25 21:11:23 2022 +0200 @@ -0,0 +1,82 @@ +/* + * page-api-jobresults.c -- /api/v?/jobresults route + * + * Copyright (c) 2021 David Demelier + * + * Permission to use, copy, modify, and/or distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ + +#include +#include +#include +#include +#include + +#include "db.h" +#include "log.h" +#include "page.h" +#include "types.h" + +static int +save(const char *json) +{ + struct jobresult res = {0}; + int ret = -1; + + json_t *doc; + json_error_t err; + + if (!(doc = json_loads(json, 0, &err))) + log_warn("api/post: invalid JSON input: %s", err.text); + else if (jobresult_from(&res, 1, doc) < 0) + log_warn("api/post: failed to decode parameters"); + else if (db_jobresult_add(&res) < 0) + log_warn("api/post: database save error"); + else + ret = 0; + + json_decref(doc); + jobresult_finish(&res); + + return ret; +} + +static void +post(struct kreq *r) +{ + if (r->fieldsz < 1) + page(r, NULL, KHTTP_400, KMIME_APP_JSON, NULL); + else if (save(r->fields[0].val) < 0) + page(r, NULL, KHTTP_500, KMIME_APP_JSON, NULL); + else { + khttp_head(r, kresps[KRESP_CONTENT_TYPE], "%s", kmimetypes[KMIME_APP_JSON]); + khttp_head(r, kresps[KRESP_STATUS], "%s", khttps[KHTTP_200]); + khttp_body(r); + khttp_free(r); + } +} + +void +page_api_v1_jobresults(struct kreq *r) +{ + assert(r); + + switch (r->method) { + case KMETHOD_POST: + post(r); + break; + default: + page(r, NULL, KHTTP_400, KMIME_APP_JSON, NULL); + break; + } +} diff -r 2cb228f23f53 -r 34cbbd215ef7 scid/page-api-jobresults.h --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/scid/page-api-jobresults.h Mon Jul 25 21:11:23 2022 +0200 @@ -0,0 +1,27 @@ +/* + * page-api-jobresults.h -- /api/v?/jobresults route + * + * Copyright (c) 2021 David Demelier + * + * Permission to use, copy, modify, and/or distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ + +#ifndef SCI_PAGE_API_JOBRESULTS_H +#define SCI_PAGE_API_JOBRESULTS_H + +struct kreq; + +void +page_api_v1_jobresults(struct kreq *); + +#endif /* !SCI_PAGE_API_JOBRESULTS_H */ diff -r 2cb228f23f53 -r 34cbbd215ef7 sciworkerd/sciworkerd.c --- a/sciworkerd/sciworkerd.c Thu Jul 21 21:55:02 2022 +0200 +++ b/sciworkerd/sciworkerd.c Mon Jul 25 21:11:23 2022 +0200 @@ -13,7 +13,7 @@ #include "types.h" #include "util.h" -#define TAG "sigworkerd: " +#define TAG "sciworkerd: " struct taskentry { struct task *task; @@ -48,7 +48,7 @@ static void stop(int sign) { - log_info(TAG "exiting on signal %d\n", sign); + log_info(TAG "exiting on signal %d", sign); run = 0; } @@ -69,7 +69,7 @@ { struct taskentry *tk; - log_info(TAG "queued job build (%d) for tag %s\n", job->id, job->tag); + log_info(TAG "queued job build (%d) for tag %s", job->id, job->tag); tk = util_calloc(1, sizeof (*tk)); tk->task = task_new(job->tag); @@ -82,7 +82,7 @@ { size_t total = 0; - for (ssize_t i = 0; i < jobsz; ++i) { + for (size_t i = 0; i < jobsz; ++i) { if (!pending(jobs[i].id)) { queue(&jobs[i]); total++; @@ -130,9 +130,7 @@ if (apic_worker_find(&req, &worker, sciworkerd.name) < 0) log_die(TAG "unable to fetch worker info: %s", req.error); - log_info("worker name: %s", worker.name); - log_info("worker description: %s", worker.desc); - + log_info("sciworkerd: worker %s (%s)", worker.name, worker.desc); apic_finish(&req); } @@ -191,10 +189,11 @@ size_t running = count(tasks); struct taskentry *entry; - while (running-- > 0 && (entry = taskpending)) { + while (running < sciworkerd.maxjobs && (entry = taskpending)) { if (start(entry) < 0) delete(entry); else { + running++; LL_DELETE(taskpending, entry); LL_APPEND(tasks, entry); } @@ -208,6 +207,7 @@ struct taskcode code; struct pollfd *fds; size_t fdsz, i = 0; + pid_t pid; int ret; /* First, read every pipes. */ @@ -239,6 +239,8 @@ /* Now wait for the task to complete. */ if (ret <= 0) { + pid = task_pid(iter->task); + if (task_wait(iter->task) < 0) log_warn(TAG "task wait error: %s", strerror(errno)); else { @@ -247,11 +249,11 @@ switch (task_status(iter->task)) { case TASKSTATUS_EXITED: log_info(TAG "task %lld exited with code %d", - (long long int)task_pid(iter->task), code.exitcode); + (long long int)pid, code.exitcode); break; case TASKSTATUS_KILLED: log_info(TAG "task %lld killed with signal %d", - (long long int)task_pid(iter->task), code.sigcode); + (long long int)pid, code.sigcode); break; default: break; @@ -259,10 +261,11 @@ } /* Remove that task and push to the outgoing queue. */ - next = iter->next; LL_DELETE(tasks, iter); LL_APPEND(taskfinished, iter); } + + iter = next; } free(fds); @@ -275,7 +278,6 @@ ghost_all(void) { struct taskentry *iter, *tmp; - time_t now; LL_FOREACH_SAFE(tasks, iter, tmp) { if (difftime(time(NULL), task_uptime(iter->task)) < sciworkerd.timeout) @@ -304,20 +306,15 @@ .worker_name = worker.name }; struct apic req; - json_t *doc; - int ret; - - doc = jobresult_to(&res, 1); - ret = apic_post(&req, doc, "%s/api/v1/jobs", sciworkerd.url); - json_decref(doc); + int ret = 0; - if (ret) + if (apic_jobresult_add(&req, &res) < 0) { log_warn(TAG "unable to publish task: %s", req.error); - else - log_info(TAG "task successfully published"); + ret = -1; + } + apic_finish(&req); jobresult_finish(&res); - apic_finish(&req); return ret; } @@ -358,6 +355,7 @@ { while (run) { fetch_jobs(); + start_all(); process_all(); ghost_all(); publish_all(); diff -r 2cb228f23f53 -r 34cbbd215ef7 sciworkerd/task.c --- a/sciworkerd/task.c Thu Jul 21 21:55:02 2022 +0200 +++ b/sciworkerd/task.c Mon Jul 25 21:11:23 2022 +0200 @@ -65,7 +65,7 @@ goto failed; if (fchmod(self->scriptfd, MODE) < 0) goto failed; - if (write(self->scriptfd, script, len) != len) + if (write(self->scriptfd, script, len) != (ssize_t)len) goto failed; return 0; @@ -107,7 +107,7 @@ break; } - return 0; + return self->child; } int @@ -205,7 +205,7 @@ if (fd->revents & POLLIN) { if ((nr = read(self->pipe[0], buf, sizeof (buf))) <= 0) return nr; - if (fwrite(buf, 1, nr, self->fp) != nr) + if (fwrite(buf, 1, nr, self->fp) != (size_t)nr) return -1; }