Mercurial > sci
changeset 3:215c0c3b3609
misc: use JSON everywhere (scictl/sciwebd)
| author | David Demelier <markand@malikania.fr> |
|---|---|
| date | Mon, 14 Jun 2021 22:08:24 +0200 |
| parents | 5fa3d2f479b2 |
| children | 9c4fea43803c |
| files | Makefile base64.c base64.h config.def.h db.c db.h http.c job.c job.h page-api-jobs.c page-api-projects.c page-api-projects.h page-api-script.c page-api-script.h page-api-workers.c page-api-workers.h project.h req.c req.h scictl.c scid.c sciworkerd.c sql/init.sql sql/job-add.sql sql/job-queue.sql sql/job-result-todo.sql sql/job-save.sql sql/job-todo.sql sql/jobresult-add.sql sql/project-add.sql sql/project-find-id.sql sql/project-get.sql sql/project-insert.sql sql/project-list.sql sql/worker-add.sql sql/worker-find-id.sql sql/worker-get.sql sql/worker-insert.sql sql/worker-list.sql types.c types.h util.c util.h worker.h |
| diffstat | 44 files changed, 1640 insertions(+), 1489 deletions(-) [+] |
line wrap: on
line diff
--- a/Makefile Thu Jun 10 10:39:21 2021 +0200 +++ b/Makefile Mon Jun 14 22:08:24 2021 +0200 @@ -20,42 +20,44 @@ include config.mk -SCID_SRCS= base64.c \ - db.c \ - job.c \ +SCID_SRCS= db.c \ util.c \ scid.c \ + types.c \ log.c \ extern/libsqlite/sqlite3.c SCID_DATA= sql/init.h \ - sql/job-queue.h \ - sql/job-result-todo.h \ - sql/job-save.h \ + sql/job-add.h \ + sql/job-todo.h \ + sql/jobresult-add.h \ + sql/project-add.h \ sql/project-find.h \ - sql/project-get.h \ - sql/project-insert.h \ - sql/worker-get.h \ + sql/project-find-id.h \ + sql/project-list.h \ + sql/worker-add.h \ sql/worker-find.h \ - sql/worker-insert.h + sql/worker-find-id.h \ + sql/worker-list.h SCID_OBJS= ${SCID_SRCS:.c=.o} SCID_DEPS= ${SCID_SRCS:.c=.d} -SCIWORKERD_SRCS= base64.c sciworkerd.c util.c log.c +SCIWORKERD_SRCS= sciworkerd.c types.c util.c log.c SCIWORKERD_OBJS= ${SCIWORKERD_SRCS:.c=.o} SCIWORKERD_DEPS= ${SCIWORKERD_SRCS:.c=.d} -SCICTL_SRCS= base64.c req.c scictl.c util.c +SCICTL_SRCS= req.c scictl.c types.c util.c SCICTL_OBJS= ${SCICTL_SRCS:.c=.o} SCICTL_DEPS= ${SCICTL_SRCS:.c=.d} -SCIWEBD_SRCS= base64.c \ - http.c \ +SCIWEBD_SRCS= http.c \ log.c \ page-api-jobs.c \ - page-api-script.c \ + page-api-projects.c \ + page-api-workers.c \ page.c \ req.c \ sciwebd.c \ + types.c \ util.c SCIWEBD_OBJS= ${SCIWEBD_SRCS:.c=.o} SCIWEBD_DEPS= ${SCIWEBD_SRCS:.c=.d} @@ -77,9 +79,6 @@ KCGI_INCS= `pkg-config --cflags kcgi` KCGI_LIBS= `pkg-config --libs kcgi` -ZSTD_INCS= `pkg-config --cflags libzstd` -ZSTD_LIBS= `pkg-config --libs libzstd` - INCS= -Iextern/libsqlite DEFS= -DVARDIR=\"${VARDIR}\" \ -DTMPDIR=\"${TMPDIR}\" @@ -87,10 +86,10 @@ .SUFFIXES: .SUFFIXES: .c .o .sql .h -all: scid scictl sciworkerd sciwebd +all: scid scictl sciwebd sciworkerd .c.o: - ${CC} ${INCS} ${DEFS} ${LIBBSD_INCS} ${KCGI_INCS} ${JANSSON_INCS} ${ZSTD_INCS} ${CFLAGS} -MMD -c $< -o $@ + ${CC} ${INCS} ${DEFS} ${LIBBSD_INCS} ${KCGI_INCS} ${JANSSON_INCS} ${CFLAGS} -MMD -c $< -o $@ .sql.h: ./bcc -sc0 $< $< > $@ @@ -108,25 +107,25 @@ ${SCID_OBJS}: config.h ${SCID_DATA} scid: ${SCID_OBJS} - ${CC} ${CFLAGS} -o $@ ${SCID_OBJS} ${LIBBSD_LIBS} ${ZSTD_LIBS} ${LDFLAGS} + ${CC} ${CFLAGS} -o $@ ${SCID_OBJS} ${LIBBSD_LIBS} ${JANSSON_LIBS} ${LDFLAGS} ${SCIWORKERD_OBJS}: config.h sciworkerd: ${SCIWORKERD_OBJS} - ${CC} ${CFLAGS} -o $@ ${SCIWORKERD_OBJS} ${LIBBSD_LIBS} ${LIBCURL_LIBS} ${JANSSON_LIBS} ${ZSTD_LIBS} ${LDFLAGS} + ${CC} ${CFLAGS} -o $@ ${SCIWORKERD_OBJS} ${LIBBSD_LIBS} ${LIBCURL_LIBS} ${JANSSON_LIBS} ${LDFLAGS} ${SCICTL_OBJS}: config.h scictl: ${SCICTL_OBJS} - ${CC} ${CFLAGS} -o $@ ${SCICTL_OBJS} ${LIBBSD_LIBS} ${ZSTD_LIBS} ${LDFLAGS} + ${CC} ${CFLAGS} -o $@ ${SCICTL_OBJS} ${LIBBSD_LIBS} ${JANSSON_LIBS} ${LDFLAGS} ${SCIWEBD_OBJS}: config.h sciwebd: ${SCIWEBD_OBJS} - ${CC} ${CFLAGS} -o $@ ${SCIWEBD_OBJS} ${LIBBSD_LIBS} ${KCGI_LIBS} ${JANSSON_LIBS} ${ZSTD_LIBS} ${LDFLAGS} + ${CC} ${CFLAGS} -o $@ ${SCIWEBD_OBJS} ${LIBBSD_LIBS} ${KCGI_LIBS} ${JANSSON_LIBS} ${LDFLAGS} clean: - rm -f bcc config.h + rm -f bcc config.h tags cscope.out rm -f scid ${SCID_OBJS} ${SCID_DEPS} ${SCID_DATA} rm -f scictl ${SCICTL_OBJS} ${SCICTL_DEPS} rm -f sciworkerd ${SCIWORKERD_OBJS} ${SCIWORKERD_DEPS}
--- a/base64.c Thu Jun 10 10:39:21 2021 +0200 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,186 +0,0 @@ -/* - * base64.h -- base64 encoding and decoding - * - * Copyright (c) 2013-2021 David Demelier <markand@malikania.fr> - * - * Permission to use, copy, modify, and/or distribute this software for any - * purpose with or without fee is hereby granted, provided that the above - * copyright notice and this permission notice appear in all copies. - * - * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - */ - -#include <assert.h> -#include <ctype.h> -#include <errno.h> -#include <string.h> - -#include "base64.h" - -int -b64_isbase64(unsigned char ch) -{ - return isalnum(ch) || ch == '+' || ch == '-' || ch == '_' || ch == '/'; -} - -int -b64_isvalid(unsigned char ch) -{ - return b64_isbase64(ch) || ch == '='; -} - -unsigned char -b64_lookup(unsigned char value) -{ - assert(value < 64); - - static const char *table = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; - - return table[value]; -} - -unsigned char -b64_rlookup(unsigned char ch) -{ - assert(b64_isbase64(ch)); - - if (ch >= '0' && ch <= '9') - return ch + 4; - if (ch >= 'A' && ch <= 'Z') - return ch - 65; - if (ch >= 'a' && ch <= 'z') - return ch - 71; - - /* '-' is base64url support. */ - return ch == '+' || ch == '-' ? 62U : 63U; -} - -size_t -b64_encode(const char *src, size_t srcsz, char *dst, size_t dstsz) -{ - assert(src); - assert(dst); - - size_t nwritten = 0; - - if (srcsz == (size_t)-1) - srcsz = strlen(src); - - while (srcsz && dstsz) { - char inputbuf[3] = {0}; - int count = 0; - - while (srcsz && count < 3) { - inputbuf[count++] = *src++; - --srcsz; - } - - if (dstsz < 4) { - errno = ERANGE; - return -1; - } - - *dst++ = b64_lookup(inputbuf[0] >> 2 & 0x3f); - *dst++ = b64_lookup((inputbuf[0] << 4 & 0x3f) | (inputbuf[1] >> 4 & 0x0f)); - - if (count < 2) - *dst++ = '='; - else - *dst++ = b64_lookup((inputbuf[1] << 2 & 0x3c) | (inputbuf[2] >> 6 & 0x03)); - - if (count < 3) - *dst++ = '='; - else - *dst++ = b64_lookup(inputbuf[2] & 0x3f); - - nwritten += 4; - dstsz -= 4; - } - - /* Not enough room to store '\0'. */ - if (dstsz == 0) { - errno = ERANGE; - return -1; - } - - *dst = '\0'; - - return nwritten; -} - -size_t -b64_decode(const char *src, size_t srcsz, char *dst, size_t dstsz) -{ - assert(src); - assert(dst); - - size_t nwritten = 0; - - if (srcsz == (size_t)-1) - srcsz = strlen(src); - - while (srcsz && dstsz) { - int i = 0, r = 3; - unsigned int inputbuf[4] = {0}; - - for (; srcsz && i < 4; i++) { - if (*src == '=') { - /* - * '=' is only allowed in last 2 characters, - * otherwise it means we need less data. - */ - if (i <= 1) - goto eilseq; - - /* Less data required. */ - --r; - } else if (!b64_isvalid(*src)) - goto eilseq; - - if (b64_isbase64(*src)) - inputbuf[i] = b64_rlookup(*src); - - ++src; - --srcsz; - } - - /* Make sure we haven't seen AB=Z as well. */ - if (i != 4 || (src[-2] == '=' && src[-1] != '=')) - goto eilseq; - if ((size_t)r >= dstsz) - goto erange; - - *dst++ = ((inputbuf[0] << 2) & 0xfc) | - ((inputbuf[1] >> 4) & 0x03); - - if (r >= 2) - *dst++ = ((inputbuf[1] << 4) & 0xf0) | ((inputbuf[2] >> 2) & 0x0f); - if (r >= 3) - *dst++ = ((inputbuf[2] << 6) & 0xc0) | (inputbuf[3] & 0x3f); - - nwritten += r; - dstsz -= r; - } - - /* Not enough room to store '\0'. */ - if (dstsz == 0) - goto erange; - - *dst = '\0'; - - return nwritten; - -eilseq: - errno = EILSEQ; - return -1; - -erange: - errno = ERANGE; - return -1; -}
--- a/base64.h Thu Jun 10 10:39:21 2021 +0200 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,53 +0,0 @@ -/* - * base64.h -- base64 encoding and decoding - * - * Copyright (c) 2013-2021 David Demelier <markand@malikania.fr> - * - * Permission to use, copy, modify, and/or distribute this software for any - * purpose with or without fee is hereby granted, provided that the above - * copyright notice and this permission notice appear in all copies. - * - * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - */ - -#ifndef BASE64_H -#define BASE64_H - -#include <stddef.h> - -#if defined(__cplusplus) -extern "C" { -#endif - -#define B64_ENCODE_LENGTH(x) (4 * ((x) / 3 + 1)) -#define B64_DECODE_LENGTH(x) (3 * ((x) / 4)) - -int -b64_isbase64(unsigned char ch); - -int -b64_isvalid(unsigned char ch); - -unsigned char -b64_lookup(unsigned char value); - -unsigned char -b64_rlookup(unsigned char ch); - -size_t -b64_encode(const char *src, size_t srcsz, char *dst, size_t dstsz); - -size_t -b64_decode(const char *src, size_t srcsz, char *dst, size_t dstsz); - -#if defined(__cplusplus) -} -#endif - -#endif /* !BASE64_H */
--- a/config.def.h Thu Jun 10 10:39:21 2021 +0200 +++ b/config.def.h Mon Jun 14 22:08:24 2021 +0200 @@ -20,14 +20,14 @@ #define SCI_CONFIG_H /* I/O limits */ -#define SCI_CONSOLE_MAX 1048576 /* Build log max */ +#define SCI_CONSOLE_MAX 4194304 /* Build log max (4MB) */ #define SCI_MSG_MAX (SCI_CONSOLE_MAX + 1024) /* Network message max. */ /* Database limits. */ -#define SCI_PROJECT_MAX 64 /* Projects allowed in database. */ -#define SCI_WORKER_MAX 32 /* Workers allowed in database. */ +#define SCI_PROJECT_MAX 64 /* Projects allowed in database. */ +#define SCI_WORKER_MAX 32 /* Workers allowed in database. */ /* Usage limits. */ -#define SCI_JOB_LIST_MAX 128 /* Jobs max list size. */ +#define SCI_JOB_LIST_MAX 128 /* Jobs max list size. */ #endif /* !SCI_CONFIG_H */
--- a/db.c Thu Jun 10 10:39:21 2021 +0200 +++ b/db.c Mon Jun 14 22:08:24 2021 +0200 @@ -1,3 +1,4 @@ +#include <sys/queue.h> #include <assert.h> #include <stdlib.h> #include <string.h> @@ -5,34 +6,117 @@ #include <sqlite3.h> #include "db.h" -#include "job.h" #include "log.h" -#include "project.h" -#include "worker.h" +#include "types.h" +#include "util.h" #include "sql/init.h" -#include "sql/job-queue.h" -#include "sql/job-result-todo.h" -#include "sql/job-save.h" -#include "sql/project-insert.h" -#include "sql/project-get.h" +#include "sql/job-add.h" +#include "sql/job-todo.h" +#include "sql/jobresult-add.h" +#include "sql/project-add.h" #include "sql/project-find.h" -#include "sql/worker-get.h" +#include "sql/project-find-id.h" +#include "sql/project-list.h" +#include "sql/worker-add.h" #include "sql/worker-find.h" -#include "sql/worker-insert.h" +#include "sql/worker-find-id.h" +#include "sql/worker-list.h" #define CHAR(v) (const char *)(v) static sqlite3 *db; -static inline void -convert_project(struct project *project, sqlite3_stmt *stmt) +struct str { + char *str; + SLIST_ENTRY(str) link; +}; + +SLIST_HEAD(strlist, str); + +static struct strlist * +strlist_new(void) +{ + struct strlist *l; + + l = util_calloc(1, sizeof (*l)); + SLIST_INIT(l); + + return l; +} + +static const char * +strlist_add(struct strlist *l, const char *text) +{ + struct str *s; + + s = util_calloc(1, sizeof (*s)); + s->str = util_strdup(text); + + SLIST_INSERT_HEAD(l, s, link); + + return s->str; +} + +static void +strlist_free(struct strlist *l) { - project->id = sqlite3_column_int64(stmt, 0); - strlcpy(project->name, CHAR(sqlite3_column_text(stmt, 1)), sizeof (project->name)); - strlcpy(project->desc, CHAR(sqlite3_column_text(stmt, 2)), sizeof (project->desc)); - strlcpy(project->url, CHAR(sqlite3_column_text(stmt, 3)), sizeof (project->url)); - strlcpy(project->script, CHAR(sqlite3_column_text(stmt, 4)), sizeof (project->script)); + struct str *s, *tmp; + + SLIST_FOREACH_SAFE(s, l, link, tmp) { + free(s->str); + free(s); + } + + SLIST_INIT(l); +} + +static inline void +convert_project(struct db_ctx *ctx, struct project *project, sqlite3_stmt *stmt) +{ + project->id = sqlite3_column_int(stmt, 0); + project->name = strlist_add(ctx->handle, CHAR(sqlite3_column_text(stmt, 1))); + project->desc = strlist_add(ctx->handle, CHAR(sqlite3_column_text(stmt, 2))); + project->url = strlist_add(ctx->handle, CHAR(sqlite3_column_text(stmt, 3))); + project->script = strlist_add(ctx->handle, CHAR(sqlite3_column_text(stmt, 4))); +} + +static int +insert(const char *sql, const char *fmt, ...) +{ + assert(sql); + assert(fmt); + + sqlite3_stmt *stmt = NULL; + va_list ap; + + if (sqlite3_prepare(db, sql, -1, &stmt, NULL) != SQLITE_OK) + return log_warn("db: %s", sqlite3_errmsg(db)), -1; + + va_start(ap, fmt); + + for (int index = 1; *fmt; ++fmt) { + switch (*fmt) { + case 'i': + sqlite3_bind_int(stmt, index++, va_arg(ap, int)); + break; + case 's': + sqlite3_bind_text(stmt, index++, va_arg(ap, const char *), -1, SQLITE_STATIC); + break; + default: + break; + } + } + + va_end(ap); + + if (sqlite3_step(stmt) != SQLITE_DONE) { + log_warn("db: %s", sqlite3_errmsg(db)); + sqlite3_finalize(stmt); + return -1; + } + + return sqlite3_last_insert_rowid(db); } int @@ -64,7 +148,7 @@ sqlite3_stmt *stmt = NULL; int ret = -1; - if (sqlite3_prepare(db, CHAR(sql_project_insert), -1, &stmt, NULL) != SQLITE_OK) + if (sqlite3_prepare(db, CHAR(sql_project_add), -1, &stmt, NULL) != SQLITE_OK) goto sqlite3_err; sqlite3_bind_text(stmt, 1, pj->name, -1, SQLITE_STATIC); @@ -86,7 +170,7 @@ } ssize_t -db_project_get(struct project *projects, size_t projectsz) +db_project_list(struct db_ctx *ctx, struct project *projects, size_t projectsz) { assert(projects); @@ -94,15 +178,16 @@ struct project *p = projects; ssize_t ret = 0; - if (sqlite3_prepare(db, CHAR(sql_project_get), -1, &stmt, NULL) != SQLITE_OK) { + if (sqlite3_prepare(db, CHAR(sql_project_list), -1, &stmt, NULL) != SQLITE_OK) { log_warn("db: %s", sqlite3_errmsg(db)); return -1; } - sqlite3_bind_int64(stmt, 1, projectsz); + sqlite3_bind_int(stmt, 1, projectsz); + ctx->handle = strlist_new(); for (; sqlite3_step(stmt) == SQLITE_ROW && (size_t)ret < projectsz; ++ret, ++p) - convert_project(p, stmt); + convert_project(ctx, p, stmt); if (stmt) sqlite3_finalize(stmt); @@ -111,13 +196,16 @@ } int -db_project_find(struct project *project) +db_project_find(struct db_ctx *ctx, struct project *project) { + assert(ctx); assert(project); sqlite3_stmt *stmt = NULL; int ret = -1; + ctx->handle = NULL; + if (sqlite3_prepare(db, CHAR(sql_project_find), -1, &stmt, NULL) != SQLITE_OK) goto sqlite3_err; @@ -127,11 +215,52 @@ goto sqlite3_err; ret = 0; - convert_project(project, stmt); + ctx->handle = strlist_new(); + convert_project(ctx, project, stmt); sqlite3_err: - if (ret < 0) + if (ret < 0) { + if (ctx->handle) + db_ctx_finish(ctx); + log_warn("db: %s", sqlite3_errmsg(db)); + } + if (stmt) + sqlite3_finalize(stmt); + + return ret; +} + +int +db_project_find_id(struct db_ctx *ctx, struct project *project) +{ + assert(ctx); + assert(project); + + sqlite3_stmt *stmt = NULL; + int ret = -1; + + ctx->handle = NULL; + + if (sqlite3_prepare(db, CHAR(sql_project_find_id), -1, &stmt, NULL) != SQLITE_OK) + goto sqlite3_err; + + sqlite3_bind_int(stmt, 1, project->id); + + if (sqlite3_step(stmt) != SQLITE_ROW) + goto sqlite3_err; + + ret = 0; + ctx->handle = strlist_new(); + convert_project(ctx, project, stmt); + +sqlite3_err: + if (ret < 0) { + if (ctx->handle) + db_ctx_finish(ctx); + + log_warn("db: %s", sqlite3_errmsg(db)); + } if (stmt) sqlite3_finalize(stmt); @@ -146,7 +275,7 @@ sqlite3_stmt *stmt = NULL; int ret = -1; - if (sqlite3_prepare(db, CHAR(sql_worker_insert), -1, &stmt, NULL) != SQLITE_OK) + if (sqlite3_prepare(db, CHAR(sql_worker_add), -1, &stmt, NULL) != SQLITE_OK) goto sqlite3_err; sqlite3_bind_text(stmt, 1, wk->name, -1, SQLITE_STATIC); @@ -166,28 +295,36 @@ } ssize_t -db_worker_get(struct worker *wk, size_t wksz) +db_worker_list(struct db_ctx *ctx, struct worker *wk, size_t wksz) { + assert(ctx); assert(wk); sqlite3_stmt *stmt = NULL; struct worker *w = wk; ssize_t ret = -1; - if (sqlite3_prepare(db, CHAR(sql_worker_get), -1, &stmt, NULL) != SQLITE_OK) + ctx->handle = NULL; + + if (sqlite3_prepare(db, CHAR(sql_worker_list), -1, &stmt, NULL) != SQLITE_OK) goto sqlite3_err; - sqlite3_bind_int64(stmt, 1, wksz); + sqlite3_bind_int(stmt, 1, wksz); + ctx->handle = strlist_new(); for (ret = 0; sqlite3_step(stmt) == SQLITE_ROW && (size_t)ret < wksz; ++ret, ++w) { - w->id = sqlite3_column_int64(stmt, 0); - strlcpy(w->name, CHAR(sqlite3_column_text(stmt, 1)), sizeof (w->name)); - strlcpy(w->desc, CHAR(sqlite3_column_text(stmt, 2)), sizeof (w->desc)); + w->id = sqlite3_column_int(stmt, 0); + w->name = strlist_add(ctx->handle, CHAR(sqlite3_column_text(stmt, 1))); + w->desc = strlist_add(ctx->handle, CHAR(sqlite3_column_text(stmt, 2))); } sqlite3_err: - if (ret < 0) + if (ret < 0) { + if (ctx->handle) + db_ctx_finish(ctx); + log_warn("db: %s", sqlite3_errmsg(db)); + } if (stmt) sqlite3_finalize(stmt); @@ -195,13 +332,16 @@ } int -db_worker_find(struct worker *w) +db_worker_find(struct db_ctx *ctx, struct worker *w) { + assert(ctx); assert(w); sqlite3_stmt *stmt = NULL; int ret = -1; + ctx->handle = NULL; + if (sqlite3_prepare(db, CHAR(sql_worker_find), -1, &stmt, NULL) != SQLITE_OK) goto sqlite3_err; @@ -211,13 +351,18 @@ goto sqlite3_err; ret = 0; - w->id = sqlite3_column_int64(stmt, 0); - strlcpy(w->name, CHAR(sqlite3_column_text(stmt, 1)), sizeof (w->name)); - strlcpy(w->desc, CHAR(sqlite3_column_text(stmt, 2)), sizeof (w->desc)); + ctx->handle = strlist_new(); + w->id = sqlite3_column_int(stmt, 0); + w->name = strlist_add(ctx->handle, CHAR(sqlite3_column_text(stmt, 1))); + w->desc = strlist_add(ctx->handle, CHAR(sqlite3_column_text(stmt, 2))); sqlite3_err: - if (ret < 0) + if (ret < 0) { + if (ctx->handle) + db_ctx_finish(ctx); + log_warn("db: %s", sqlite3_errmsg(db)); + } if (stmt) sqlite3_finalize(stmt); @@ -225,61 +370,37 @@ } int -db_job_queue(struct job *job) +db_worker_find_id(struct db_ctx *ctx, struct worker *w) { - assert(job); + assert(ctx); + assert(w); sqlite3_stmt *stmt = NULL; int ret = -1; - if (sqlite3_prepare(db, CHAR(sql_job_queue), -1, &stmt, NULL) != SQLITE_OK) + ctx->handle = NULL; + + if (sqlite3_prepare(db, CHAR(sql_worker_find_id), -1, &stmt, NULL) != SQLITE_OK) goto sqlite3_err; - sqlite3_bind_text(stmt, 1, job->tag, -1, SQLITE_STATIC); - sqlite3_bind_int64(stmt, 2, job->project.id); + sqlite3_bind_int(stmt, 1, w->id); - if (sqlite3_step(stmt) != SQLITE_DONE) + if (sqlite3_step(stmt) != SQLITE_ROW) goto sqlite3_err; - job->id = sqlite3_last_insert_rowid(db); ret = 0; + ctx->handle = strlist_new(); + w->id = sqlite3_column_int(stmt, 0); + w->name = strlist_add(ctx->handle, CHAR(sqlite3_column_text(stmt, 1))); + w->desc = strlist_add(ctx->handle, CHAR(sqlite3_column_text(stmt, 2))); sqlite3_err: - if (ret < 0) - log_warn("db: %s", sqlite3_errmsg(db)); - if (stmt) - sqlite3_finalize(stmt); - - return ret; -} + if (ret < 0) { + if (ctx->handle) + db_ctx_finish(ctx); -ssize_t -db_job_result_todo(struct job_result *re, size_t resz, int64_t worker_id) -{ - assert(re); - - sqlite3_stmt *stmt = NULL; - ssize_t ret = 0; - - if (sqlite3_prepare(db, CHAR(sql_job_result_todo), -1, &stmt, NULL) != SQLITE_OK) { log_warn("db: %s", sqlite3_errmsg(db)); - return -1; } - - sqlite3_bind_int64(stmt, 1, worker_id); - sqlite3_bind_int64(stmt, 2, resz); - - while (sqlite3_step(stmt) == SQLITE_ROW && (size_t)ret++ < resz) { - memset(re, 0, sizeof (*re)); - re->job.id = sqlite3_column_int64(stmt, 0); - strlcpy(re->job.tag, CHAR(sqlite3_column_text(stmt, 1)), - sizeof (re->job.tag)); - strlcpy(re->job.project.name, CHAR(sqlite3_column_text(stmt, 2)), - sizeof (re->job.project.name)); - - ++re; - }; - if (stmt) sqlite3_finalize(stmt); @@ -287,35 +408,53 @@ } int -db_job_save(struct job_result *r) +db_job_add(struct job *job) +{ + assert(job); + + job->id = insert(CHAR(sql_job_add), "si", job->tag, job->project_id); + + return job->id < 0 ? -1 : 0; +} + +ssize_t +db_job_todo(struct db_ctx *ctx, struct job *jobs, size_t jobsz, int worker_id) +{ + assert(ctx); + assert(jobs); + + sqlite3_stmt *stmt = NULL; + ssize_t ret = 0; + + if (sqlite3_prepare(db, CHAR(sql_job_todo), -1, &stmt, NULL) != SQLITE_OK) { + log_warn("db: %s", sqlite3_errmsg(db)); + return -1; + } + + sqlite3_bind_int(stmt, 1, worker_id); + sqlite3_bind_int(stmt, 2, jobsz); + ctx->handle = strlist_new(); + + while (sqlite3_step(stmt) == SQLITE_ROW && (size_t)ret++ < jobsz) { + jobs->id = sqlite3_column_int(stmt, 0); + jobs->tag = strlist_add(ctx->handle, CHAR(sqlite3_column_text(stmt, 1))); + jobs++->project_id = sqlite3_column_int(stmt, 2); + }; + + sqlite3_finalize(stmt); + + return ret; +} + +int +db_jobresult_add(struct jobresult *r) { assert(r); - sqlite3_stmt *stmt = NULL; - int ret = -1; - - if (sqlite3_prepare(db, CHAR(sql_job_save), -1, &stmt, NULL) != SQLITE_OK) - goto sqlite3_err; - - sqlite3_bind_int64(stmt, 1, r->job.id); - sqlite3_bind_int64(stmt, 2, r->worker.id); - sqlite3_bind_int(stmt, 3, r->status); - sqlite3_bind_int(stmt, 4, r->retcode); - sqlite3_bind_text(stmt, 5, r->console, -1, SQLITE_STATIC); + r->id = insert(CHAR(sql_jobresult_add), "iiis", r->job_id, + r->worker_id, r->exitcode, r->log); - if (sqlite3_step(stmt) != SQLITE_DONE) - goto sqlite3_err; - - ret = 0; - r->id = sqlite3_last_insert_rowid(db); - -sqlite3_err: - if (ret < 0) - log_warn("db: %s", sqlite3_errmsg(db)); - if (stmt) - sqlite3_finalize(stmt); - - return ret; + return r->id < 0 ? -1 : 0; } void @@ -326,3 +465,12 @@ db = NULL; } } + +void +db_ctx_finish(struct db_ctx *ctx) +{ + if (ctx->handle) { + strlist_free(ctx->handle); + ctx->handle = NULL; + } +}
--- a/db.h Thu Jun 10 10:39:21 2021 +0200 +++ b/db.h Mon Jun 14 22:08:24 2021 +0200 @@ -3,44 +3,56 @@ #include <sys/types.h> #include <stddef.h> -#include <stdint.h> struct project; struct worker; struct job; -struct job_result; +struct jobresult; + +struct db_ctx { + void *handle; +}; int db_open(const char *); int +db_job_add(struct job *); + +ssize_t +db_job_todo(struct db_ctx *, struct job *, size_t, int); + +int +db_jobresult_add(struct jobresult *); + +int db_project_add(struct project *); ssize_t -db_project_get(struct project *, size_t); +db_project_list(struct db_ctx *, struct project *, size_t); int -db_project_find(struct project *); +db_project_find(struct db_ctx *, struct project *); + +int +db_project_find_id(struct db_ctx *, struct project *); int db_worker_add(struct worker *); ssize_t -db_worker_get(struct worker *, size_t); - -int -db_worker_find(struct worker *); +db_worker_list(struct db_ctx *, struct worker *, size_t); int -db_job_queue(struct job *); - -ssize_t -db_job_result_todo(struct job_result *, size_t, int64_t); +db_worker_find(struct db_ctx *, struct worker *); int -db_job_save(struct job_result *); +db_worker_find_id(struct db_ctx *, struct worker *); void db_finish(void); +void +db_ctx_finish(struct db_ctx *); + #endif /* !SCI_DB_H */
--- a/http.c Thu Jun 10 10:39:21 2021 +0200 +++ b/http.c Mon Jun 14 22:08:24 2021 +0200 @@ -11,7 +11,8 @@ #include "log.h" #include "page.h" #include "page-api-jobs.h" -#include "page-api-script.h" +#include "page-api-projects.h" +#include "page-api-workers.h" #include "req.h" enum page { @@ -26,27 +27,17 @@ const char *prefix; void (*handler)(struct kreq *); } apis[] = { - { "v1/jobs", page_api_v1_jobs }, - { "v1/script", page_api_v1_script }, - { NULL, NULL } + { "v1/jobs", page_api_v1_jobs }, + { "v1/projects", page_api_v1_projects }, + { "v1/workers", page_api_v1_workers }, + { NULL, NULL } }; - if (req_connect(VARDIR "/run/sci.sock") < 0) { - page(req, NULL, KHTTP_500, KMIME_TEXT_HTML, "pages/500.html"); - return; - } - - for (size_t i = 0; apis[i].prefix; ++i) { - if (strncmp(req->path, apis[i].prefix, strlen(apis[i].prefix)) == 0) { - apis[i].handler(req); - goto finish; - } - } + for (size_t i = 0; apis[i].prefix; ++i) + if (strncmp(req->path, apis[i].prefix, strlen(apis[i].prefix)) == 0) + return apis[i].handler(req); page(req, NULL, KHTTP_404, KMIME_TEXT_HTML, "pages/404.html"); - -finish: - req_finish(); } static const char *pages[] = {
--- a/job.c Thu Jun 10 10:39:21 2021 +0200 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,32 +0,0 @@ -/* - * job.c -- job description and result - * - * Copyright (c) 2021 David Demelier <markand@malikania.fr> - * - * Permission to use, copy, modify, and/or distribute this software for any - * purpose with or without fee is hereby granted, provided that the above - * copyright notice and this permission notice appear in all copies. - * - * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - */ - -#include <assert.h> -#include <stdlib.h> -#include <string.h> - -#include "job.h" - -void -job_result_finish(struct job_result *res) -{ - assert(res); - - free(res->console); - memset(res, 0, sizeof (*res)); -}
--- a/job.h Thu Jun 10 10:39:21 2021 +0200 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,53 +0,0 @@ -/* - * job.h -- job description and result - * - * Copyright (c) 2021 David Demelier <markand@malikania.fr> - * - * Permission to use, copy, modify, and/or distribute this software for any - * purpose with or without fee is hereby granted, provided that the above - * copyright notice and this permission notice appear in all copies. - * - * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - */ - -#ifndef SCI_JOB_H -#define SCI_JOB_H - -#include <stdint.h> - -#define JOB_TAG_MAX 128 - -#include "project.h" -#include "worker.h" - -enum job_status { - JOB_STATUS_TODO, - JOB_STATUS_SUCCESS, - JOB_STATUS_FAIL -}; - -struct job { - int64_t id; - struct project project; - char tag[JOB_TAG_MAX]; -}; - -struct job_result { - int64_t id; - struct job job; - struct worker worker; - enum job_status status; - int retcode; - char *console; -}; - -void -job_result_finish(struct job_result *); - -#endif /* !SCI_JOB_H */
--- a/page-api-jobs.c Thu Jun 10 10:39:21 2021 +0200 +++ b/page-api-jobs.c Mon Jun 14 22:08:24 2021 +0200 @@ -8,97 +8,78 @@ #include <jansson.h> #include "config.h" -#include "job.h" #include "log.h" +#include "page-api-jobs.h" #include "page.h" #include "req.h" +#include "types.h" #include "util.h" static void -list(struct kreq *r, const struct job_result *jobs, size_t jobsz) +list(struct kreq *r, const struct job *jobs, size_t jobsz) { - json_t *array, *obj; + json_t *doc; + char *dump; - array = json_array(); + doc = job_to(jobs, jobsz); + dump = json_dumps(doc, JSON_COMPACT); + + khttp_puts(r, dump); + free(dump); + json_decref(doc); +} - for (size_t i = 0; i < jobsz; ++i) { - obj = json_object(); - json_object_set(obj, "id", json_integer(jobs[i].job.id)); - json_object_set(obj, "tag", json_string(jobs[i].job.tag)); - json_object_set(obj, "project", json_string(jobs[i].job.project.name)); - json_array_append(array, obj); - } +static int +save(const char *json) +{ + struct req req = {0}; + struct jobresult res = {0}; + int ret = -1; + + json_t *doc; + json_error_t err; - khttp_puts(r, json_dumps(array, JSON_COMPACT)); + if (!(doc = json_loads(json, 0, &err))) + log_warn("api/post: invalid JSON input: %s", err.text); + else if (jobresult_from(&res, 1, doc) < 0) + log_warn("api/post: failed to decode parameters"); + else if ((req = req_jobresult_add(&res)).status) + log_warn("api/post: save error: %s", strerror(req.status)); + else + ret = 0; + + json_decref(doc); + req_finish(&req); + + return ret; } static void get(struct kreq *r) { struct req req; - struct job_result jobs[SCI_JOB_LIST_MAX]; + struct job jobs[SCI_JOB_LIST_MAX]; size_t jobsz = UTIL_SIZE(jobs); const char *worker = util_basename(r->path); - if ((req = req_job_list(jobs, &jobsz, worker)).status) + if ((req = req_job_todo(jobs, &jobsz, worker)).status) page(r, NULL, KHTTP_500, KMIME_APP_JSON, NULL); else { khttp_head(r, kresps[KRESP_CONTENT_TYPE], "%s", kmimetypes[KMIME_APP_JSON]); khttp_head(r, kresps[KRESP_STATUS], "%s", khttps[KHTTP_200]); khttp_body(r); list(r, jobs, jobsz); + req_finish(&req); khttp_free(r); } } -static int -parse(struct job_result *res, const char *json) -{ - json_t *doc, *code, *id, *retcode; - json_error_t err; - - if (!(doc = json_loads(json, 0, &err))) - return log_warn("api/post: invalid JSON input: %s", err.text), -1; - if (!json_is_object(doc) || - !json_is_string((code = json_object_get(doc, "code"))) || - !json_is_integer((id = json_object_get(doc, "id"))) || - !json_is_integer((retcode = json_object_get(doc, "retcode")))) { - log_warn("api/post: invalid JSON input"); - json_decref(doc); - return -1; - } - - res->job.id = json_integer_value(id); - res->retcode = json_integer_value(retcode); - res->console = util_strdup(json_string_value(code)); - json_decref(doc); - - return 0; -} - -static int -save(struct job_result *res) -{ - struct req req; - - if ((req = req_job_save(res)).status) { - log_warn("api/post: save error: %s", strerror(req.status)); - return -1; - } - - return 0; -} - static void post(struct kreq *r) { - struct job_result res = {0}; - const char *worker = util_basename(r->path); - - strlcpy(res.worker.name, worker, sizeof (res.worker.name)); - log_info("data=%s", r->fields[0].key); - - if (r->fieldsz < 1 || parse(&res, r->fields[0].key) || save(&res) < 0) + if (r->fieldsz < 1) + page(r, NULL, KHTTP_400, KMIME_APP_JSON, NULL); + else if (save(r->fields[0].key) < 0) page(r, NULL, KHTTP_500, KMIME_APP_JSON, NULL); else { khttp_head(r, kresps[KRESP_CONTENT_TYPE], "%s", kmimetypes[KMIME_APP_JSON]); @@ -106,8 +87,6 @@ khttp_body(r); khttp_free(r); } - - free(res.console); } void
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/page-api-projects.c Mon Jun 14 22:08:24 2021 +0200 @@ -0,0 +1,118 @@ +#include <assert.h> + +#include "config.h" +#include "page-api-projects.h" +#include "page.h" +#include "req.h" +#include "types.h" +#include "util.h" + +static void +list(struct kreq *r, const struct project *projects, size_t projectsz) +{ + struct json_t *doc; + char *dump; + + doc = project_to(projects, projectsz); + dump = json_dumps(doc, JSON_COMPACT); + + khttp_puts(r, dump); + free(dump); + json_decref(doc); +} + +static void +push(struct kreq *r, const struct project *p) +{ + struct json_t *json; + char *dump; + + json = project_to(p, 1); + dump = json_dumps(json, JSON_COMPACT); + + khttp_head(r, kresps[KRESP_CONTENT_TYPE], "%s", kmimetypes[KMIME_APP_JSON]); + khttp_head(r, kresps[KRESP_STATUS], "%s", khttps[KHTTP_200]); + khttp_body(r); + khttp_puts(r, dump); + khttp_free(r); + + free(dump); + json_decref(json); +} + +static void +get_one(struct kreq *r, const char *name) +{ + struct project project; + struct req req; + + if ((req = req_project_find(&project, name)).status) + page(r, NULL, KHTTP_500, KMIME_APP_JSON, NULL); + else + push(r, &project); +} + +static void +get_one_id(struct kreq *r, int id) +{ + struct project project; + struct req req; + + if ((req = req_project_find_id(&project, id)).status) + page(r, NULL, KHTTP_500, KMIME_APP_JSON, NULL); + else + push(r, &project); +} + +static void +get_all(struct kreq *r) +{ + struct project projects[SCI_PROJECT_MAX]; + struct req req; + size_t projectsz = UTIL_SIZE(projects); + + if ((req = req_project_list(projects, &projectsz)).status) + page(r, NULL, KHTTP_500, KMIME_APP_JSON, NULL); + else { + khttp_head(r, kresps[KRESP_CONTENT_TYPE], "%s", kmimetypes[KMIME_APP_JSON]); + khttp_head(r, kresps[KRESP_STATUS], "%s", khttps[KHTTP_200]); + khttp_body(r); + list(r, projects, projectsz); + req_finish(&req); + khttp_free(r); + } +} + +static void +get(struct kreq *r) +{ + char name[128]; + int id; + + if (sscanf(r->path, "v1/projects/%d", &id) == 1) + get_one_id(r, id); + else if (sscanf(r->path, "v1/projects/%127s", name) == 1) + get_one(r, name); + else + get_all(r); +} + +void +page_api_v1_projects(struct kreq *r) +{ + assert(r); + + switch (r->method) { + case KMETHOD_GET: + get(r); + break; +#if 0 + case KMETHOD_POST: + post(r); + break; +#endif + default: + page(r, NULL, KHTTP_400, KMIME_APP_JSON, NULL); + break; + } +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/page-api-projects.h Mon Jun 14 22:08:24 2021 +0200 @@ -0,0 +1,9 @@ +#ifndef SCI_PAGE_API_PROJECTS_H +#define SCI_PAGE_API_PROJECTS_H + +struct kreq; + +void +page_api_v1_projects(struct kreq *); + +#endif /* !SCI_PAGE_API_PROJECTS_H */
--- a/page-api-script.c Thu Jun 10 10:39:21 2021 +0200 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,56 +0,0 @@ -#include <sys/types.h> -#include <assert.h> -#include <stdarg.h> -#include <stdint.h> - -#include <kcgi.h> -#include <jansson.h> - -#include "config.h" -#include "page.h" -#include "req.h" -#include "util.h" - -static void -content(struct kreq *r, const char *code) -{ - json_t *doc; - - doc = json_object(); - json_object_set(doc, "code", json_string(code)); - khttp_puts(r, json_dumps(doc, JSON_COMPACT)); - json_decref(doc); -} - -static void -get(struct kreq *r) -{ - struct req req; - char script[SCI_MSG_MAX]; - const char *project = util_basename(r->path); - - if ((req = req_script_get(project, script, sizeof (script))).status) - page(r, NULL, KHTTP_500, KMIME_APP_JSON, NULL); - else { - khttp_head(r, kresps[KRESP_CONTENT_TYPE], "%s", kmimetypes[KMIME_APP_JSON]); - khttp_head(r, kresps[KRESP_STATUS], "%s", khttps[KHTTP_200]); - khttp_body(r); - content(r, script); - khttp_free(r); - } -} - -void -page_api_v1_script(struct kreq *r) -{ - assert(r); - - switch (r->method) { - case KMETHOD_GET: - get(r); - break; - default: - page(r, NULL, KHTTP_400, KMIME_APP_JSON, NULL); - break; - } -}
--- a/page-api-script.h Thu Jun 10 10:39:21 2021 +0200 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,9 +0,0 @@ -#ifndef SCI_PAGE_API_SCRIPT_H -#define SCI_PAGE_API_SCRIPT_H - -struct kreq; - -void -page_api_v1_script(struct kreq *); - -#endif /* !SCI_PAGE_API_SCRIPT_H */
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/page-api-workers.c Mon Jun 14 22:08:24 2021 +0200 @@ -0,0 +1,113 @@ +#include <assert.h> + +#include "config.h" +#include "page-api-workers.h" +#include "page.h" +#include "req.h" +#include "types.h" +#include "util.h" + +static void +list(struct kreq *r, const struct worker *workers, size_t workersz) +{ + struct json_t *doc; + char *dump; + + doc = worker_to(workers, workersz); + dump = json_dumps(doc, JSON_COMPACT); + + khttp_puts(r, dump); + free(dump); + json_decref(doc); +} + +static void +push(struct kreq *r, const struct worker *p) +{ + struct json_t *json; + char *dump; + + json = worker_to(p, 1); + dump = json_dumps(json, JSON_COMPACT); + + khttp_head(r, kresps[KRESP_CONTENT_TYPE], "%s", kmimetypes[KMIME_APP_JSON]); + khttp_head(r, kresps[KRESP_STATUS], "%s", khttps[KHTTP_200]); + khttp_body(r); + khttp_puts(r, dump); + khttp_free(r); + + free(dump); + json_decref(json); +} + +static void +get_one(struct kreq *r, const char *name) +{ + struct worker worker; + struct req req; + + if ((req = req_worker_find(&worker, name)).status) + page(r, NULL, KHTTP_500, KMIME_APP_JSON, NULL); + else + push(r, &worker); +} + +static void +get_one_id(struct kreq *r, int id) +{ + struct worker worker; + struct req req; + + if ((req = req_worker_find_id(&worker, id)).status) + page(r, NULL, KHTTP_500, KMIME_APP_JSON, NULL); + else + push(r, &worker); +} + +static void +get_all(struct kreq *r) +{ + struct worker workers[SCI_PROJECT_MAX]; + struct req req; + size_t workersz = UTIL_SIZE(workers); + + if ((req = req_worker_list(workers, &workersz)).status) + page(r, NULL, KHTTP_500, KMIME_APP_JSON, NULL); + else { + khttp_head(r, kresps[KRESP_CONTENT_TYPE], "%s", kmimetypes[KMIME_APP_JSON]); + khttp_head(r, kresps[KRESP_STATUS], "%s", khttps[KHTTP_200]); + khttp_body(r); + list(r, workers, workersz); + req_finish(&req); + khttp_free(r); + } +} + +static void +get(struct kreq *r) +{ + char name[128]; + int id; + + if (sscanf(r->path, "v1/workers/%d", &id) == 1) + get_one_id(r, id); + else if (sscanf(r->path, "v1/workers/%127s", name) == 1) + get_one(r, name); + else + get_all(r); +} + +void +page_api_v1_workers(struct kreq *r) +{ + assert(r); + + switch (r->method) { + case KMETHOD_GET: + get(r); + break; + default: + page(r, NULL, KHTTP_400, KMIME_APP_JSON, NULL); + break; + } +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/page-api-workers.h Mon Jun 14 22:08:24 2021 +0200 @@ -0,0 +1,9 @@ +#ifndef SCI_PAGE_API_WORKERS_H +#define SCI_PAGE_API_WORKERS_H + +struct kreq; + +void +page_api_v1_workers(struct kreq *); + +#endif /* !SCI_PAGE_API_WORKERS_H */
--- a/project.h Thu Jun 10 10:39:21 2021 +0200 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,19 +0,0 @@ -#ifndef SCI_PROJECT_H -#define SCI_PROJECT_H - -#include <limits.h> -#include <stdint.h> - -#define PROJECT_NAME_MAX 32 -#define PROJECT_DESC_MAX 256 -#define PROJECT_URL_MAX 128 - -struct project { - int64_t id; - char name[PROJECT_NAME_MAX]; - char desc[PROJECT_DESC_MAX]; - char url[PROJECT_URL_MAX]; - char script[PATH_MAX]; -}; - -#endif /* !SCI_PROJECT_H */
--- a/req.c Thu Jun 10 10:39:21 2021 +0200 +++ b/req.c Mon Jun 14 22:08:24 2021 +0200 @@ -4,80 +4,26 @@ #include <assert.h> #include <err.h> #include <errno.h> +#include <limits.h> #include <stdarg.h> #include <stdio.h> #include <stdlib.h> #include <string.h> #include <unistd.h> -#include "job.h" -#include "project.h" +#include "config.h" #include "req.h" +#include "types.h" #include "util.h" -#include "worker.h" - -static int sock; - -static struct req -ask(const char *fmt, ...) -{ - assert(fmt); - - struct req res = {0}; - char buf[1024], *p; - va_list ap; - ssize_t bufsz = 0, nr; - - va_start(ap, fmt); - vsnprintf(buf, sizeof (buf), fmt, ap); - va_end(ap); - - if (strlcat(buf, "\r\n\r\n", sizeof (buf)) >= sizeof (buf)) { - res.status = EMSGSIZE; - return res; - } - - if (send(sock, buf, strlen(buf), MSG_NOSIGNAL) < 0) { - res.status = errno; - return res; - } - while ((nr = recv(sock, buf + bufsz, sizeof (buf) - bufsz, 0)) > 0) { - bufsz += nr; - - if ((size_t)bufsz >= sizeof (buf)) { - bufsz = 0; - res.status = EMSGSIZE; - break; - } - } - - buf[bufsz] = '\0'; - - /* Remove final '\r\n\r\n' */ - if ((p = strstr(buf, "\r\n\r\n"))) - *p = '\0'; +static char path[PATH_MAX] = VARDIR "/run/sci.sock"; - /* Check and remove status. */ - if (strncmp(buf, "ERR", 3) == 0) - res.status = -1; - if ((p = strchr(buf, '\n'))) - ++p; - else - p = buf; - - strlcat(res.msg, p, sizeof (res.msg)); - - return res; -} - -int -req_connect(const char *path) +static int +attach(const char *path) { - assert(path); - struct sockaddr_un sun; struct timeval tv = { .tv_sec = 3 }; + int sock; if ((sock = socket(PF_LOCAL, SOCK_STREAM, 0)) < 0) return -1; @@ -86,67 +32,117 @@ strlcpy(sun.sun_path, path, sizeof (sun.sun_path)); if (setsockopt(sock, SOL_SOCKET, SO_RCVTIMEO, &tv, sizeof (tv)) < 0 || - setsockopt(sock, SOL_SOCKET, SO_SNDTIMEO, &tv, sizeof (tv)) < 0) + setsockopt(sock, SOL_SOCKET, SO_SNDTIMEO, &tv, sizeof (tv)) < 0 || + connect(sock, (const struct sockaddr *)&sun, sizeof (sun)) < 0) { + close(sock); return -1; - if (connect(sock, (const struct sockaddr *)&sun, sizeof (sun)) < 0) - return -1; + } + + return sock; +} + +static struct req +exchange(const char *cmd, json_t *doc) +{ + char *json, *response, buf[BUFSIZ]; + struct req res = {0}; + int sock; + ssize_t nr; + FILE *fp; + + if ((sock = attach(path)) < 0) + goto connect_err; + + json_object_set(doc, "cmd", json_string(cmd)); + json = json_dumps(doc, JSON_COMPACT); + + if (send(sock, json, strlen(json), MSG_NOSIGNAL) <= 0 || + send(sock, "\r\n", 2, MSG_NOSIGNAL) <= 0) + goto send_err; + + response = util_malloc(SCI_MSG_MAX); + fp = util_fmemopen(response, SCI_MSG_MAX, "w"); + setbuf(fp, NULL); - return 0; + while ((nr = recv(sock, buf, sizeof (buf), MSG_NOSIGNAL)) > 0) + if (fwrite(buf, 1, nr, fp) != (size_t)nr) + goto io_err; + + if (!(res.msg = json_loads(response, 0, NULL))) + errno = EILSEQ; + else + errno = 0; + + /* TODO: generic error handling */ + +io_err: + fclose(fp); + free(response); + +send_err: + close(sock); + free(json); + +connect_err: + res.status = errno; + json_decref(doc); + + return res; +} + +void +req_set_path(const char *newpath) +{ + assert(newpath); + + strlcpy(path, newpath, sizeof (path)); } struct req -req_job_queue(const struct job *job) +req_job_add(const struct job *job) { assert(job); - return ask("job-queue %s|%s", job->project.name, job->tag); + return exchange("job-add", job_to(job, 1)); } struct req -req_job_list(struct job_result *jobs, size_t *jobsz, const char *project) +req_job_todo(struct job *jobs, size_t *jobsz, const char *worker) { assert(jobs); assert(jobsz); + assert(worker); - struct req req; - char fmt[128], *token, *p = req.msg; - size_t tot = 0; - - if ((req = ask("job-list %s", project)).status) - return req; + struct req st; + struct worker wk; - snprintf(fmt, sizeof (fmt), "%%zd|%%%zu[^|]|%%%zu[^|]\n", - sizeof (jobs->job.tag), sizeof (jobs->job.project.name)); + /* + * Retrieve worker id first, we don't need JSON document to stay + * because we only refer to the id field. + */ + if ((st = req_worker_find(&wk, worker)).status) + return st; - while ((token = strtok_r(p, "\n", &p)) && tot < *jobsz) { - if (sscanf(token, fmt, &jobs->job.id, jobs->job.tag, jobs->job.project.name) == 3) { - ++jobs; - ++tot; - } + req_finish(&st); + + if ((st = exchange("job-todo", json_pack("{si}", "worker_id", wk.id))).status) + return st; + + if ((*jobsz = job_from(jobs, *jobsz, st.msg)) == (size_t)-1) { + json_decref(st.msg); + st.msg = NULL; + st.status = errno; } - *jobsz = tot; - - return req; + return st; } struct req -req_job_save(const struct job_result *res) +req_jobresult_add(const struct jobresult *res) { assert(res); - char *b64; - struct req req = {0}; - - if (!(b64 = util_zbase64_enc(res->console))) - req.status = errno; - else { - req = ask("job-save %lld|%s|%s|%s|%s", (long long int)res->job.id, - res->worker.name, res->status, res->retcode, b64); - free(b64); - } - - return req; + return exchange("jobresult-add", jobresult_to(res, 1)); } struct req @@ -154,35 +150,66 @@ { assert(p); - return ask("project-add %s|%s|%s|%s", p->name, p->desc, p->url, p->script); + return exchange("project-add", project_to(p, 1)); +} + +struct req +req_project_find(struct project *p, const char *name) +{ + assert(p); + assert(name); + + struct req st; + + if ((st = exchange("project-find", json_pack("{ss}", "name", name))).status) + return st; + + if (project_from(p, 1, st.msg) < 0) { + json_decref(st.msg); + st.msg = NULL; + st.status = errno; + } + + return st; } struct req -req_project_list(struct project *pc, size_t *pcsz) +req_project_find_id(struct project *p, int id) { - assert(pc); - assert(pcsz); + assert(p); - struct req req; - char fmt[128], *token, *p = req.msg; - size_t tot = 0; + struct req st; - if ((req = ask("project-list")).status) - return req; - - snprintf(fmt, sizeof (fmt), "%%%zu[^|]|%%%zu[^|]|%%%zu[^|]|%%%zu[^\n]\n", - sizeof (pc->name), sizeof (pc->desc), sizeof (pc->url), sizeof (pc->script)); + if ((st = exchange("project-find-id", json_pack("{si}", "id", id))).status) + return st; - while ((token = strtok_r(p, "\n", &p)) && tot < *pcsz) { - if (sscanf(token, fmt, pc->name, pc->desc, pc->url, pc->script) == 4) { - ++pc; - ++tot; - } + if (project_from(p, 1, st.msg) < 0) { + json_decref(st.msg); + st.msg = NULL; + st.status = errno; } - *pcsz = tot; + return st; +} + +struct req +req_project_list(struct project *projects, size_t *projectsz) +{ + assert(projects); + assert(projectsz); + + struct req st; - return req; + if ((st = exchange("project-list", json_object())).status) + return st; + + if ((*projectsz = project_from(projects, *projectsz, st.msg)) == (size_t)-1) { + json_decref(st.msg); + st.msg = NULL; + st.status = errno; + } + + return st; } struct req @@ -190,7 +217,44 @@ { assert(w); - return ask("worker-add %s|%s", w->name, w->desc); + return exchange("worker-add", worker_to(w, 1)); +} + +struct req +req_worker_find(struct worker *wk, const char *name) +{ + assert(wk); + assert(name); + + struct req st; + + if ((st = exchange("worker-find", json_pack("{ss}", "name", name))).status) + return st; + if (worker_from(wk, 1, st.msg) < 0) { + json_decref(st.msg); + st.msg = NULL; + st.status = errno; + } + + return st; +} + +struct req +req_worker_find_id(struct worker *wk, int id) +{ + assert(wk); + + struct req st; + + if ((st = exchange("worker-find-id", json_pack("{si}", "id", id))).status) + return st; + if (worker_from(wk, 1, st.msg) < 0) { + json_decref(st.msg); + st.msg = NULL; + st.status = errno; + } + + return st; } struct req @@ -199,54 +263,25 @@ assert(wk); assert(wksz); - struct req req; - char fmt[128], *token, *p = req.msg; - size_t tot = 0; + struct req st; - if ((req = ask("worker-list")).status) - return req; + if ((st = exchange("worker-list", json_object())).status) + return st; - snprintf(fmt, sizeof (fmt), "%%%zu[^|]|%%%zu[^\n]\n", - sizeof (wk->name), sizeof (wk->desc)); - - while ((token = strtok_r(p, "\n", &p)) && tot < *wksz) { - if (sscanf(token, fmt, wk->name, wk->desc) == 2) { - wk++; - tot++; - } + if ((*wksz = worker_from(wk, *wksz, st.msg)) == (size_t)-1) { + json_decref(st.msg); + st.msg = NULL; + st.status = errno; } - *wksz = tot; - - return req; -} - -struct req -req_script_get(const char *project, char *out, size_t outsz) -{ - assert(out); - - struct req req; - char *script; - - if ((req = ask("script-get %s", project)).status) - return req; - if (!(script = util_zbase64_dec(req.msg))) { - req.status = EINVAL; - return req; - } - - if (strlcpy(out, script, outsz) >= outsz) - req.status = errno; - - free(script); - - return req; + return st; } void -req_finish(void) +req_finish(struct req *st) { - close(sock); - sock = 0; + if (st->msg) + json_decref(st->msg); + + memset(st, 0, sizeof (*st)); }
--- a/req.h Thu Jun 10 10:39:21 2021 +0200 +++ b/req.h Mon Jun 14 22:08:24 2021 +0200 @@ -3,44 +3,55 @@ #include <stddef.h> +#include <jansson.h> + +struct job; +struct jobresult; +struct project; +struct worker; + struct req { int status; - char msg[1024]; + json_t *msg; }; -struct worker; -struct project; -struct job; -struct job_result; - -int -req_connect(const char *); +void +req_set_path(const char *path); struct req -req_job_queue(const struct job *); +req_job_add(const struct job *); struct req -req_job_list(struct job_result *, size_t *, const char *); +req_job_todo(struct job *, size_t *, const char *); struct req -req_job_save(const struct job_result *); +req_jobresult_add(const struct jobresult *); struct req req_project_add(const struct project *); struct req -req_project_list(struct project *, size_t *); +req_project_find(struct project *, const char *); struct req -req_script_get(const char *, char *, size_t); +req_project_find_id(struct project *, int); + +struct req +req_project_list(struct project *, size_t *); struct req req_worker_add(const struct worker *); struct req +req_worker_find(struct worker *, const char *); + +struct req +req_worker_find_id(struct worker *, int); + +struct req req_worker_list(struct worker *, size_t *); void -req_finish(void); +req_finish(struct req *); #endif /* !SCI_REQ_H */
--- a/scictl.c Thu Jun 10 10:39:21 2021 +0200 +++ b/scictl.c Mon Jun 14 22:08:24 2021 +0200 @@ -1,4 +1,3 @@ -#define _BSD_SOURCE #include <err.h> #include <stdio.h> #include <stdlib.h> @@ -7,11 +6,9 @@ #include <unistd.h> #include "config.h" -#include "project.h" -#include "job.h" #include "req.h" +#include "types.h" #include "util.h" -#include "worker.h" noreturn static void usage(void) @@ -23,12 +20,11 @@ noreturn static void help(void) { - fprintf(stderr, "usage: %s job-queue project tag\n", getprogname()); - fprintf(stderr, " %s job-list worker\n", getprogname()); - fprintf(stderr, " %s job-save id worker status retcode console\n", getprogname()); + fprintf(stderr, "usage: %s job-add project tag\n", getprogname()); + fprintf(stderr, " %s job-todo worker\n", getprogname()); + fprintf(stderr, " %s jobresult-add id worker exitcode console\n", getprogname()); fprintf(stderr, " %s project-add name desc url script\n", getprogname()); fprintf(stderr, " %s project-list\n", getprogname()); - fprintf(stderr, " %s script-get project\n", getprogname()); fprintf(stderr, " %s worker-add name desc\n", getprogname()); fprintf(stderr, " %s worker-list\n", getprogname()); exit(0); @@ -38,8 +34,7 @@ readfile(const char *path) { FILE *fp, *str; - static char console[SCI_MSG_MAX]; - char buf[BUFSIZ], *ret = console; + char buf[BUFSIZ], *console; size_t nr; if (strcmp(path, "-") == 0) @@ -47,90 +42,157 @@ else if (!(fp = fopen(path, "r"))) err(1, "%s", path); - if (!(str = fmemopen(console, sizeof (console), "w"))) + console = util_calloc(1, SCI_MSG_MAX); + + if (!(str = fmemopen(console, SCI_MSG_MAX, "w"))) err(1, "fmemopen"); while ((nr = fread(buf, 1, sizeof (buf), fp)) > 0) fwrite(buf, 1, nr, str); - if ((ferror(fp) && !feof(fp)) || (ferror(str) && !feof(str))) - ret = NULL; + if ((ferror(fp) && !feof(fp)) || (ferror(str) && !feof(str))) { + free(console); + console = NULL; + } fclose(str); fclose(fp); - return ret; + return console; } static struct req -cmd_job_queue(int argc, char **argv) +cmd_job_add(int argc, char **argv) { struct job job = {0}; + struct project project = {0}; + struct req rp, rj; if (argc < 2) usage(); - strlcpy(job.project.name, argv[0], sizeof (job.project.name)); - strlcpy(job.tag, argv[1], sizeof (job.tag)); + if ((rp = req_project_find(&project, argv[0])).status) + return rp; - return req_job_queue(&job); + job.project_id = project.id; + job.tag = argv[1]; + + rj = req_job_add(&job); + req_finish(&rp); + + return rj; } static struct req -cmd_job_list(int argc, char **argv) +cmd_job_todo(int argc, char **argv) { - struct job_result jobs[SCI_JOB_LIST_MAX]; - size_t jobsz = UTIL_SIZE(jobs); - struct req req; + struct project projects[SCI_PROJECT_MAX] = {0}; + struct job jobs[SCI_JOB_LIST_MAX] = {0}; + struct req rp, rj; + size_t projectsz = UTIL_SIZE(projects), jobsz = UTIL_SIZE(jobs); if (argc < 1) usage(); - if ((req = req_job_list(jobs, &jobsz, argv[0])).status) - return req; + /* First retrieve projects for a better listing. */ + if ((rp = req_project_list(projects, &projectsz)).status) + return rp; - printf("%-16s%-16s%s\n", "ID", "TAG", "PROJECT"); + if ((rj = req_job_todo(jobs, &jobsz, argv[0])).status) { + req_finish(&rp); + return rj; + } for (size_t i = 0; i < jobsz; ++i) { - printf("%-16lld%-16s%s\n", (long long int)jobs[i].job.id, - jobs[i].job.tag, jobs[i].job.project.name); + const char *project = "unknown"; + + /* Find project if exists (it should). */ + for (size_t i = 0; i < projectsz; ++i) { + if (projects[i].id == jobs[i].project_id) { + project = projects[i].name; + break; + } + } + + printf("%-16s%d\n", "id:", jobs[i].id); + printf("%-16s%s\n", "tag:", jobs[i].tag); + printf("%-16s%s\n", "project:", project); + + if (i + 1 < jobsz) + printf("\n"); } - return req; + req_finish(&rp); + + return rj; } static struct req -cmd_job_save(int argc, char **argv) +cmd_jobresult_add(int argc, char **argv) { - struct job_result res = {0}; + struct jobresult res = {0}; + struct worker wk = {0}; + struct req rw, rj; + char *log; if (argc < 5) usage(); - res.job.id = strtoll(argv[0], NULL, 10); - res.status = strtoll(argv[2], NULL, 10); - res.retcode = strtoll(argv[3], NULL, 10); - res.console = readfile(argv[4]); - strlcpy(res.worker.name, argv[1], sizeof (res.worker.name)); + /* Find worker id. */ + if ((rw = req_worker_find(&wk, argv[1])).status) + return rw; - return req_job_save(&res); + res.job_id = strtoll(argv[0], NULL, 10); + res.exitcode = strtoll(argv[2], NULL, 10); + res.worker_id = wk.id; + res.log = log = readfile(argv[3]); + rj = req_jobresult_add(&res); + + free(log); + req_finish(&rw); + + return rj; } static struct req cmd_project_add(int argc, char **argv) { - struct project pc; + struct project pc = {0}; + struct req res; + char *script; if (argc < 4) usage(); - memset(&pc, 0, sizeof (pc)); - strlcpy(pc.name, argv[0], sizeof (pc.name)); - strlcpy(pc.desc, argv[1], sizeof (pc.desc)); - strlcpy(pc.url, argv[2], sizeof (pc.url)); - strlcpy(pc.script, argv[3], sizeof (pc.script)); + pc.name = argv[0]; + pc.desc = argv[1]; + pc.url = argv[2]; + pc.script = script = readfile(argv[3]); + res = req_project_add(&pc); + + free(script); + + return res; +} - return req_project_add(&pc); +static struct req +cmd_project_find(int argc, char **argv) +{ + struct project project = {0}; + struct req req; + + if (argc < 1) + usage(); + if ((req = req_project_find(&project, argv[0])).status) + return req; + + printf("%-16s%s\n", "name:", project.name); + printf("%-16s%s\n", "desc:", project.desc); + printf("%-16s%s\n", "url:", project.url); + printf("\n"); + printf("%s", project.script); + + return req; } static struct req @@ -139,43 +201,21 @@ (void)argc; (void)argv; - struct project pc[SCI_PROJECT_MAX]; + struct project projects[SCI_PROJECT_MAX] = {0}; struct req req; - size_t pcsz = UTIL_SIZE(pc); + size_t projectsz = UTIL_SIZE(projects); - memset(pc, 0, sizeof (pc)); - - if ((req = req_project_list(pc, &pcsz)).status) + if ((req = req_project_list(projects, &projectsz)).status) return req; - printf("%-16s%-24s%-20s%s\n", "NAME", "DESCRIPTION", "URL", "SCRIPT"); - - for (size_t i = 0; i < pcsz; ++i) - printf("%-16s%-24s%-20s%s\n", pc[i].name, pc[i].desc, - pc[i].url, pc[i].script); - - return req; -} + for (size_t i = 0; i < projectsz; ++i) { + printf("%-16s%s\n", "name:", projects[i].name); + printf("%-16s%s\n", "desc:", projects[i].desc); + printf("%-16s%s\n", "url:", projects[i].url); -static struct req -cmd_script_get(int argc, char **argv) -{ - char script[SCI_MSG_MAX]; - struct req req; - - if (argc < 1) - usage(); - if ((req = req_script_get(argv[0], script, sizeof (script))).status) - return req; - - printf("%s", script); - - /* - * Don't break up the terminal output if the script does not contain a - * final new line. - */ - if (script[strlen(script) - 1] != '\n') - printf("\n"); + if (i + 1 < projectsz) + printf("\n"); + } return req; } @@ -183,14 +223,13 @@ static struct req cmd_worker_add(int argc, char **argv) { - struct worker wk; + struct worker wk = {0}; if (argc < 2) usage(); - memset(&wk, 0, sizeof (wk)); - strlcpy(wk.name, argv[0], sizeof (wk.name)); - strlcpy(wk.desc, argv[1], sizeof (wk.desc)); + wk.name = argv[0]; + wk.desc = argv[1]; return req_worker_add(&wk); } @@ -208,10 +247,13 @@ if ((req = req_worker_list(wk, &wksz)).status) return req; - printf("%-16s%s\n", "NAME", "DESCRIPTION"); + for (size_t i = 0; i < wksz; ++i) { + printf("%-16s%s\n", "name:", wk[i].name); + printf("%-16s%s\n", "desc:", wk[i].desc); - for (size_t i = 0; i < wksz; ++i) - printf("%-16s%s\n", wk[i].name, wk[i].desc); + if (i + 1 < wksz) + printf("\n"); + } return req; } @@ -220,12 +262,12 @@ const char *name; struct req (*exec)(int, char **); } commands[] = { - { "job-queue", cmd_job_queue }, - { "job-list", cmd_job_list }, - { "job-save", cmd_job_save }, + { "job-add", cmd_job_add }, + { "job-todo", cmd_job_todo }, + { "jobresult-add", cmd_jobresult_add }, { "project-add", cmd_project_add }, + { "project-find", cmd_project_find }, { "project-list", cmd_project_list }, - { "script-get", cmd_script_get }, { "worker-add", cmd_worker_add }, { "worker-list", cmd_worker_list }, { NULL, NULL } @@ -234,7 +276,6 @@ int main(int argc, char **argv) { - const char *sock = VARDIR "/run/sci.sock"; int ch, cmdfound = 0; setprogname("scictl"); @@ -242,7 +283,7 @@ while ((ch = getopt(argc, argv, "s:")) != -1) { switch (ch) { case 's': - sock = optarg; + req_set_path(optarg); break; default: break; @@ -256,8 +297,6 @@ usage(); if (strcmp(argv[0], "help") == 0) help(); - if (req_connect(sock) < 0) - err(1, "%s", sock); for (size_t i = 0; commands[i].name; ++i) { struct req res; @@ -266,15 +305,16 @@ res = commands[i].exec(--argc, ++argv); cmdfound = 1; +#if 0 if (res.status) warnx("%s", res.msg); +#endif + req_finish(&res); break; } } if (!cmdfound) errx(1, "abort: command %s not found", argv[0]); - - req_finish(); }
--- a/scid.c Thu Jun 10 10:39:21 2021 +0200 +++ b/scid.c Mon Jun 14 22:08:24 2021 +0200 @@ -23,6 +23,7 @@ #include <err.h> #include <errno.h> #include <fcntl.h> +#include <limits.h> #include <stdio.h> #include <stdlib.h> #include <stdnoreturn.h> @@ -31,17 +32,15 @@ #include "config.h" #include "db.h" -#include "job.h" #include "log.h" -#include "project.h" +#include "types.h" #include "util.h" -#include "worker.h" static char dbpath[PATH_MAX] = VARDIR "/db/sci/sci.db"; static char scpath[PATH_MAX] = VARDIR "/run/sci.sock"; static int sc; -noreturn static void +noreturn static inline void usage(void) { fprintf(stderr, "usage: %s [-d database]\n", getprogname()); @@ -76,384 +75,274 @@ err(1, "unable to open database"); } -static size_t -split(char *line, char **args, size_t max) +static int +answer(int fd, json_t *json) { - char *token, *ptr; - size_t i = 0; + char *str; + int ret = 0; + + str = json_dumps(json, JSON_COMPACT); - for (ptr = line; i < max && (token = strtok_r(ptr, "|", &ptr)); ++i) - args[i] = token; + if (send(fd, str, strlen(str), 0) < 0 || send(fd, "\r\n", 2, 0) < 0) + ret = -1; - return i; + free(str); + json_decref(json); + + return ret; } -static int -answer(int fd, const char *fmt, ...) +static inline int +ok(int fd) { - char buf[1024] = {0}; + return answer(fd, json_pack("{ss}", "status", "ok")); +} + +static inline int +error(int fd, const char *fmt, ...) +{ + char buf[1024]; va_list ap; va_start(ap, fmt); vsnprintf(buf, sizeof (buf), fmt, ap); - strlcat(buf, "\r\n\r\n", sizeof (buf)); va_end(ap); - send(fd, buf, strlen(buf), MSG_NOSIGNAL); + return answer(fd, json_pack("{ss ss}", + "status", "error", + "error", buf + )); +} - return 0; +static int +cmd_job_add(int fd, json_t *doc) +{ + struct job job = {0}; + + if (job_from(&job, 1, doc) < 0) + return EINVAL; + if (db_job_add(&job) < 0) + return error(fd, "unable to create job"); + + log_info("queued new job (%d), with tag %s", job.id, job.tag); + + return ok(fd); } static int -ok(int fd) +cmd_job_todo(int fd, json_t *doc) { - send(fd, "OK", 2, MSG_NOSIGNAL); + struct db_ctx ctx; + struct job jobs[SCI_JOB_LIST_MAX]; + ssize_t jobsz; + int worker_id, ret; - return 0; + if (json_unpack(doc, "{si}", "worker_id", &worker_id) < 0) + return EINVAL; + if ((jobsz = db_job_todo(&ctx, jobs, UTIL_SIZE(jobs), worker_id)) < 0) + return error(fd, "unable to retrieve list"); + + ret = answer(fd, job_to(jobs, jobsz)); + db_ctx_finish(&ctx); + + return ret; } -/* - * Request - * ------- - * - * job-queue project|tag - * - * Errors - * ------ - * - * - Already pending - * - Internal database error - */ static int -cmd_job_queue(int fd, char *cmd) +cmd_jobresult_add(int fd, json_t *doc) { - char *args[2] = {0}; - struct job job = {0}; - struct project project; + struct jobresult res = {0}; + + if (jobresult_from(&res, 1, doc) < 0) + return EINVAL; + if (db_jobresult_add(&res) < 0) + return error(fd, "unable to create job result"); + + log_info("insert new result (%d), with job_id %d (exitcode=%d)", + res.id, res.job_id, res.exitcode); - if (split(cmd, args, 2) != 2) { - log_warn("invalid job-queue invocation"); - return EINVAL; - } - - strlcpy(project.name, args[0], sizeof (project.name)); + return ok(fd); +} - if (db_project_find(&project) < 0) { - log_warn("project %s not found", args[0]); - return ENOENT; - } +static int +cmd_project_add(int fd, json_t *doc) +{ + struct project proj = {0}; - job.project.id = project.id; - strlcpy(job.tag, args[1], sizeof (job.tag)); + if (project_from(&proj, 1, doc) < 0) + return EINVAL; + if (db_project_add(&proj) < 0) + return error(fd, "unable to create project"); - if (db_job_queue(&job) < 0) - return answer(fd, "ERR unable to create job"); - - log_info("queued new job (%lld), for project %s with tag %s", - (long long int)job.id, args[0], args[1]); + log_info("created new project (%d) %s", proj.id, proj.name); return ok(fd); } static int -cmd_job_list(int fd, char *cmd) +cmd_project_find(int fd, json_t *doc) { - char *args[1] = {0}, buf[SCI_MSG_MAX]; - struct job_result jobs[SCI_JOB_LIST_MAX]; - struct worker worker; - ssize_t n; - FILE *fp; - - if (split(cmd, args, 1) != 1) { - log_warn("invalid job-list invocation"); - return EINVAL; - } - if (!(fp = fmemopen(buf, sizeof (buf), "w"))) - return ENOMEM; - - strlcpy(worker.name, args[0], sizeof (worker.name)); + struct db_ctx ctx; + struct project proj = {0}; + int ret; - if (db_worker_find(&worker) < 0) { - log_warn("worker %s not found", args[0]); - return ENOENT; - } - - if ((n = db_job_result_todo(jobs, UTIL_SIZE(jobs), worker.id)) < 0) - return answer(fd, "ERR unable to retrieve jobs list"); + if (json_unpack(doc, "{ss}", "name", &proj.name) < 0) + return EINVAL; + if (db_project_find(&ctx, &proj) < 0) + return error(fd, "unable to retrieve project"); - fprintf(fp, "OK\n"); + ret = answer(fd, project_to(&proj, 1)); + db_ctx_finish(&ctx); - for (ssize_t i = 0; i < n; ++i) - fprintf(fp, "%lld|%s|%s\n", (long long int)jobs[i].job.id, - jobs[i].job.tag, jobs[i].job.project.name); - - fclose(fp); - - return answer(fd, "%s", buf); + return ret; } -/* - * Request - * ------- - * - * job-save id|worker|status|retcode|console - */ static int -cmd_job_save(int fd, char *cmd) +cmd_project_find_id(int fd, json_t *doc) { - char *args[5] = {0}; - struct job_result res; - - if (split(cmd, args, 5) != 5) { - log_warn("invalid job-save invocation"); - return EINVAL; - } - - strlcpy(res.worker.name, args[1], sizeof (res.worker.name)); + struct db_ctx ctx; + struct project proj = {0}; + int ret; - if (db_worker_find(&res.worker) < 0) { - log_warn("worker %s not found", args[1]); - return ENOENT; - } - - res.job.id = strtoll(args[0], NULL, 10); - res.status = strtoll(args[2], NULL, 10); - res.retcode = strtoll(args[3], NULL, 10); - res.console = util_zbase64_dec(args[4]); - - if (!res.console) { - log_warn("failed to decode console data"); + if (json_unpack(doc, "{si}", "id", &proj.id) < 0) return EINVAL; - } + if (db_project_find_id(&ctx, &proj) < 0) + return error(fd, "unable to retrieve project"); - if (db_job_save(&res) < 0) { - log_warn("failed to save job result"); - return EINVAL; - } + ret = answer(fd, project_to(&proj, 1)); + db_ctx_finish(&ctx); - log_info("save job info (%lld, status=%d retcode=%d)", - (long long int)res.id, res.status, res.retcode); - free(res.console); - - return ok(fd); + return ret; } -/* - * Request - * ------- - * - * project-add name|desc|url|script - * - * Errors - * ------ - * - * - Already exists - * - Internal database error - */ static int -cmd_project_add(int fd, char *cmd) +cmd_project_list(int fd, json_t *doc) { - char *args[4] = {0}; - struct project proj = {0}; + (void)doc; - if (split(cmd, args, 4) != 4) { - log_warn("invalid project-add invocation"); + struct db_ctx ctx; + struct project projects[SCI_PROJECT_MAX]; + ssize_t projectsz; + int ret; + + if ((projectsz = db_project_list(&ctx, projects, UTIL_SIZE(projects))) < 0) return EINVAL; - } + + ret = answer(fd, project_to(projects, projectsz)); + db_ctx_finish(&ctx); - strlcpy(proj.name, args[0], sizeof (proj.name)); - strlcpy(proj.desc, args[1], sizeof (proj.desc)); - strlcpy(proj.url, args[2], sizeof (proj.url)); - strlcpy(proj.script, args[3], sizeof (proj.script)); + return ret; +} - if (db_project_add(&proj) < 0) - return answer(fd, "ERR unable to create project"); +static int +cmd_worker_add(int fd, json_t *doc) +{ + struct worker wk = {0}; + + if (worker_from(&wk, 1, doc) < 0) + return EINVAL; + if (db_worker_add(&wk) <0) + return error(fd, "unable to create worker"); + + log_info("created new worker (%d) %s", wk.id, wk.name); return ok(fd); } -/* - * Request - * ------- - * - * project-list - * - * Answer - * ------ - * - * OK - * name|desc|url|script - * ... - * - * Errors - * ------ - * - * - Internal database error - */ static int -cmd_project_list(int fd, char *cmd) -{ - (void)cmd; - - char buf[SCI_MSG_MAX]; - struct project projects[SCI_PROJECT_MAX]; - ssize_t np; - FILE *fp; - - if ((np = db_project_get(projects, SCI_PROJECT_MAX)) < 0) - return EINVAL; - if (!(fp = fmemopen(buf, sizeof (buf), "w"))) - return ENOMEM; - - fprintf(fp, "OK\n"); - - for (ssize_t i = 0; i < np; ++i) - fprintf(fp, "%s|%s|%s|%s\n", projects[i].name, projects[i].desc, - projects[i].url, projects[i].script); - - fclose(fp); - - return answer(fd, "%s", buf); -} - -/* - * Request - * ------- - * - * worker-add name|desc - * - * Errors - * ------ - * - * - Internal database error - */ -static int -cmd_worker_add(int fd, char *cmd) +cmd_worker_find(int fd, json_t *doc) { - char *lines[2]; - struct worker wk; - - if (split(cmd, lines, 2) != 2) { - log_warn("invalid worker-add invocation"); - return EINVAL; - } - - strlcpy(wk.name, lines[0], sizeof (wk.name)); - strlcpy(wk.desc, lines[1], sizeof (wk.desc)); - - if (db_worker_add(&wk) < 0) - return answer(fd, "ERR unable to create project"); - - return ok(fd); -} + struct worker wk = {0}; + struct db_ctx ctx; + int ret; -/* - * Request - * ------- - * - * worker-list - * - * Answer - * ------ - * - * OK - * name|desc - * ... - * - * Errors - * ------ - * - * - Internal database error - */ -static int -cmd_worker_list(int fd, char *cmd) -{ - (void)cmd; + if (json_unpack(doc, "{ss}", "name", &wk.name) < 0) + return EINVAL; + if (db_worker_find(&ctx, &wk) < 0) + return error(fd, "unable to retrieve worker"); - char buf[SCI_MSG_MAX]; - struct worker wk[SCI_WORKER_MAX]; - ssize_t np; - FILE *fp; + ret = answer(fd, worker_to(&wk, 1)); + db_ctx_finish(&ctx); - if ((np = db_worker_get(wk, SCI_WORKER_MAX)) < 0) - return EINVAL; - if (!(fp = fmemopen(buf, sizeof (buf), "w"))) - return ENOMEM; - - fprintf(fp, "OK\n"); - - for (ssize_t i = 0; i < np; ++i) - fprintf(fp, "%s|%s\n", wk[i].name, wk[i].desc); - - fclose(fp); - - return answer(fd, "%s", buf); + return ret; } static int -cmd_script_get(int fd, char *cmd) +cmd_worker_find_id(int fd, json_t *doc) { - char buf[SCI_MSG_MAX], *b64; - struct project project = {0}; - int filed, ret; - ssize_t nr; + struct worker wk = {0}; + struct db_ctx ctx; + int ret; - strlcpy(project.name, cmd, sizeof (project.name)); + if (json_unpack(doc, "{si}", "id", &wk.id) < 0) + return EINVAL; + if (db_worker_find_id(&ctx, &wk) < 0) + return error(fd, "unable to retrieve worker"); + + ret = answer(fd, worker_to(&wk, 1)); + db_ctx_finish(&ctx); - if (db_project_find(&project) < 0) - return ENOENT; - if ((filed = open(project.script, O_RDONLY)) < 0) - return errno; - if ((nr = read(filed, buf, sizeof (buf) - 1)) <= 0) { - close(filed); - return errno; - } + return ret; +} + +static int +cmd_worker_list(int fd, json_t *doc) +{ + (void)doc; - buf[nr] = 0; - close(filed); + struct db_ctx ctx; + struct worker workers[SCI_WORKER_MAX]; + ssize_t workersz; + int ret; - if (!(b64 = util_zbase64_enc(buf))) - return errno; + if ((workersz = db_worker_list(&ctx, workers, UTIL_SIZE(workers))) < 0) + return error(fd, "unable to retrieve worker list"); - ret = answer(fd, "OK\n%s", b64); - free(b64); + ret = answer(fd, worker_to(workers, workersz)); + db_ctx_finish(&ctx); return ret; } static void -dispatch(int fd, char *cmd) +dispatch(int fd, const char *msg) { - assert(cmd); - static const struct { const char *name; - int (*exec)(int, char *); + int (*exec)(int, json_t *); } cmds[] = { - { "job-queue", cmd_job_queue }, - { "job-list", cmd_job_list }, - { "job-save", cmd_job_save }, + { "job-add", cmd_job_add }, + { "job-todo", cmd_job_todo }, + { "jobresult-add", cmd_jobresult_add }, { "project-add", cmd_project_add }, + { "project-find", cmd_project_find }, + { "project-find-id", cmd_project_find_id }, { "project-list", cmd_project_list }, { "worker-add", cmd_worker_add }, + { "worker-find", cmd_worker_find }, + { "worker-find-id", cmd_worker_find_id }, { "worker-list", cmd_worker_list }, - { "script-get", cmd_script_get }, { NULL, NULL } }; - for (size_t i = 0; cmds[i].name; ++i) { - size_t len = strlen(cmds[i].name); - int ret; - - if (strncmp(cmds[i].name, cmd, len) == 0) { - cmd += len; + json_t *doc, *cmd; + json_error_t err; - while (*cmd && isspace(*cmd)) - cmd++; + if (!(doc = json_loads(msg, 0, &err))) + return log_warn("json error: %s", err.text); + if (!json_is_string((cmd = json_object_get(doc, "cmd")))) { + json_decref(doc); + return log_warn("json invalid"); + } - if ((ret = cmds[i].exec(fd, cmd)) != 0) - answer(fd, "ERR %s", strerror(ret)); + for (size_t i = 0; cmds[i].name; ++i) { + if (strcmp(cmds[i].name, json_string_value(cmd)) == 0) { + int ret = 0; + if ((ret = cmds[i].exec(fd, doc)) != 0) + error(fd, strerror(errno)); + + json_decref(doc); break; } } @@ -463,9 +352,10 @@ run(void) { int client; - char buf[1024] = {0}, *endl; - ssize_t bufsz = 0, nr; - struct timeval tv = { .tv_sec = 3 }; + FILE *fp; + char *msg, *endl, buf[BUFSIZ]; + ssize_t nr; + struct timeval tv = { .tv_sec = 30 }; if ((client = accept(sc, NULL, 0)) < 0) err(1, "accept"); @@ -474,24 +364,22 @@ if (setsockopt(client, SOL_SOCKET, SO_SNDTIMEO, &tv, sizeof (tv)) < 0) err(1, "setsockopt"); - while (!strstr(buf, "\r\n\r\n") && (nr = recv(client, buf + bufsz, sizeof (buf) - bufsz, 0)) > 0) { - bufsz += nr; + fp = util_fmemopen((msg = util_calloc(1, SCI_MSG_MAX)), SCI_MSG_MAX, "w"); + setbuf(fp, NULL); - if ((size_t)bufsz >= sizeof (buf)) - goto end; - } + while (!strstr(msg, "\r\n") && (nr = recv(client, buf, sizeof (buf), 0)) > 0) + if (fwrite(buf, 1, nr, fp) != (size_t)nr) + warn("fwrite"); if (nr < 0) warn("recv"); - - buf[bufsz] = '\0'; - - if ((endl = strstr(buf, "\r\n\r\n"))) + if ((endl = strstr(msg, "\r\n"))) *endl = 0; - dispatch(client, buf); -end: + fclose(fp); + dispatch(client, msg); close(client); + free(msg); } int
--- a/sciworkerd.c Thu Jun 10 10:39:21 2021 +0200 +++ b/sciworkerd.c Mon Jun 14 22:08:24 2021 +0200 @@ -18,11 +18,12 @@ #include <jansson.h> #include "config.h" -#include "job.h" #include "log.h" -#include "project.h" +#include "types.h" #include "util.h" +#define TAG_MAX 256 + enum taskst { TASKST_PENDING, /* not started yet. */ TASKST_RUNNING, /* currently running. */ @@ -34,8 +35,10 @@ enum taskst status; pid_t child; int pipe[2]; - int retcode; - struct job job; + int exitcode; + int job_id; + int project_id; + char job_tag[TAG_MAX]; char out[SCI_CONSOLE_MAX]; char script[PATH_MAX]; int scriptfd; @@ -70,6 +73,7 @@ }; static struct tasks tasks = TAILQ_HEAD_INITIALIZER(tasks); +static struct worker worker; #if 0 static int sigpipe[2]; @@ -109,7 +113,7 @@ static void destroy(struct task *tk) { - log_debug("destroying task %lld", tk->job.id); + log_debug("destroying task %d", tk->job_id); if (tk->pipe[0]) close(tk->pipe[0]); @@ -141,11 +145,12 @@ dup2(tk->pipe[1], STDERR_FILENO); close(tk->pipe[0]); close(tk->pipe[1]); - log_debug("spawn: running process (%lld) %s", tk->child, tk->script); + log_debug("spawn: running process (%lld) %s", + (long long int)tk->child, tk->script); tk->status = TASKST_RUNNING; - if (execl(tk->script, tk->script, tk->job.tag, NULL) < 0) { + if (execl(tk->script, tk->script, tk->job_tag, NULL) < 0) { tk->status = TASKST_PENDING; log_warn("exec %s: %s", tk->script, strerror(errno)); exit(0); @@ -213,7 +218,7 @@ log_debug("process %lld completed", (long long int)sinfo->si_pid); close(tk->pipe[1]); tk->status = TASKST_COMPLETED; - tk->retcode = status; + tk->exitcode = status; tk->pipe[1] = 0; } @@ -228,73 +233,25 @@ #endif } -static void -init(void) -{ - struct sigaction sa; - - sa.sa_flags = SA_SIGINFO; - sa.sa_sigaction = complete; - sigemptyset(&sa.sa_mask); - - if (sigaction(SIGCHLD, &sa, NULL) < 0) - err(1, "sigaction"); - - log_open("sciworkerd"); - -#if 0 - if (pipe(sigpipe) < 0) - err(1, "pipe"); - if ((flags = fcntl(sigpipe[1], F_GETFL, 0)) < 0 || - fcntl(sigpipe[1], F_SETFL, flags | O_NONBLOCK) < 0) - err(1, "fcntl"); -#endif -} - -static struct fds -prepare(void) -{ - struct fds fds = {0}; - struct task *tk; - size_t i = 0; - - TAILQ_FOREACH(tk, &tasks, link) - if (tk->status == TASKST_RUNNING || tk->status == TASKST_COMPLETED) - fds.listsz++; - - fds.list = util_calloc(fds.listsz, sizeof (*fds.list)); - -#if 0 - fds.list[0].fd = sigpipe[0]; - fds.list[0].events = POLLIN; -#endif - printf("fd => %zu\n", fds.listsz); - - TAILQ_FOREACH(tk, &tasks, link) { - if (tk->status == TASKST_RUNNING || tk->status == TASKST_COMPLETED) { - printf("adding %d to pollin\n", tk->pipe[0]); - fds.list[i].fd = tk->pipe[0]; - fds.list[i++].events = POLLIN | POLLPRI; - } - } - - return fds; -} - static const char * uploadenc(const struct task *tk) { - static char json[SCI_MSG_MAX]; - json_t *object; + json_t *doc; + + struct jobresult res = {0}; + char *dump; - object = json_object(); - json_object_set(object, "code", json_string(tk->out)); - json_object_set(object, "id", json_integer(tk->job.id)); - json_object_set(object, "retcode", json_integer(tk->retcode)); - strlcpy(json, json_dumps(object, JSON_COMPACT), sizeof (json)); - json_decref(object); + res.job_id = tk->job_id; + res.exitcode = tk->exitcode; + res.log = tk->out; + res.worker_id = worker.id; - return json; + doc = jobresult_to(&res, 1); + dump = json_dumps(doc, JSON_COMPACT); + + json_decref(doc); + + return dump; } static size_t @@ -306,12 +263,16 @@ return w; } -static const char * +static json_t * get(const char *topic, const char *url) { CURL *curl; CURLcode code; - static char buf[SCI_MSG_MAX]; + + json_t *doc; + json_error_t error; + + char buf[SCI_MSG_MAX]; long status; FILE *fp; @@ -320,9 +281,6 @@ if (!(fp = fmemopen(buf, sizeof (buf), "w"))) err(1, "fmemopen"); -#if 0 - curl_easy_setopt(curl, CURLOPT_URL, makeurl("api/v1/script/%s", tk->job.project.name)); -#endif curl_easy_setopt(curl, CURLOPT_URL, url); curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L); curl_easy_setopt(curl, CURLOPT_TIMEOUT, 3L); @@ -341,8 +299,10 @@ return log_warn("%s: %s", topic, curl_easy_strerror(code)), NULL; if (status != 200) return log_warn("%s: unexpected status code %ld", topic, status), NULL; + if (!(doc = json_loads(buf, 0, &error))) + return log_warn("%s: %s", topic, error.text), NULL; - return buf; + return doc; } static size_t @@ -363,7 +323,7 @@ long status; curl = curl_easy_init(); - curl_easy_setopt(curl, CURLOPT_URL, makeurl("api/v1/jobs/%s", config.worker)); + curl_easy_setopt(curl, CURLOPT_URL, makeurl("api/v1/jobs")); curl_easy_setopt(curl, CURLOPT_TIMEOUT, 3L); curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L); curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, silent); @@ -393,83 +353,66 @@ static inline void finished(struct task *tk) { - log_info("task %d: completed with exit code %d", tk->child, tk->retcode); + log_info("task %d: completed with exit code %d", tk->child, tk->exitcode); printf("== OUTPUT ==\n"); puts(tk->out); upload(tk); } static inline int -pending(int64_t id) +pending(int id) { struct task *t; TAILQ_FOREACH(t, &tasks, link) - if (t->job.id == id) + if (t->job_id == id) return 1; return 0; } static void -push(int64_t id, const char *tag, const char *project) +queue(int id, int project_id, const char *tag) { struct task *tk; - log_info("queued job build (%lld) for project %s, tag %s\n", id, project, tag); + log_info("queued job build (%d) for tag %s\n", id, tag); tk = util_calloc(1, sizeof (*tk)); - tk->job.id = id; - strlcpy(tk->job.tag, tag, sizeof (tk->job.tag)); - strlcpy(tk->job.project.name, project, sizeof (tk->job.project.name)); + tk->job_id = id; + tk->project_id = project_id; + strlcpy(tk->job_tag, tag, sizeof (tk->job_tag)); TAILQ_INSERT_TAIL(&tasks, tk, link); } static void -merge(const char *str) +merge(json_t *doc) { - json_t *array, *obj, *id, *tag, *project; - json_error_t err; - size_t i; + struct job jobs[SCI_JOB_LIST_MAX]; + ssize_t jobsz; - if (!(array = json_loads(str, 0, &err))) { - log_warn("fetch: failed to decode JSON: %s", err.text); - return; - } - if (!json_is_array(array)) - goto invalid; - - json_array_foreach(array, i, obj) { - if (!json_is_object(obj) || - !json_is_integer((id = json_object_get(obj, "id"))) || - !json_is_string((tag = json_object_get(obj, "tag"))) || - !json_is_string((project = json_object_get(obj, "project")))) - goto invalid; - - if (!pending(json_integer_value(id))) - push(json_integer_value(id), json_string_value(tag), - json_string_value(project)); + if ((jobsz = job_from(jobs, UTIL_SIZE(jobs), doc)) < 0) + log_warn("fetchjobs: %s", strerror(errno)); + else { + for (ssize_t i = 0; i < jobsz; ++i) { + if (!pending(jobs[i].id)) + queue(jobs[i].id, jobs[i].project_id, jobs[i].tag); + } } - json_decref(array); - - return; - -invalid: - log_warn("fetch: invalid JSON input"); - json_decref(array); + json_decref(doc); } static void fetchjobs(void) { - const char *json; + json_t *doc; - if (!(json = get("fetch", makeurl("api/v1/jobs/%s", config.worker)))) + if (!(doc = get("fetch", makeurl("api/v1/jobs/%s", config.worker)))) log_warn("unable to retrieve jobs"); else - merge(json); + merge(doc); } /* @@ -514,26 +457,22 @@ } static int -extract(struct task *tk, const char *json) +extract(struct task *tk, json_t *doc) { - json_t *doc, *code; - json_error_t err; + struct project proj; size_t len; - if (!(doc = json_loads(json, 0, &err))) { - log_warn("fetchscript: failed to decode JSON: %s", err.text); + if (project_from(&proj, 1, doc) < 0) { + json_decref(doc); + log_warn("fetchproject: %s", strerror(errno)); return -1; } - if (!json_is_object(doc) || - !json_is_string((code = json_object_get(doc, "code")))) - goto invalid; + + len = strlen(proj.script); - len = strlen(json_string_value(code)); - - if ((size_t)write(tk->scriptfd, json_string_value(code), len) != len) { - log_warn("fetchscript: %s", strerror(errno)); + if ((size_t)write(tk->scriptfd, proj.script, len) != len) { json_decref(doc); - + log_warn("fetchproject: %s", strerror(errno)); return -1; } @@ -542,36 +481,31 @@ tk->scriptfd = 0; return 0; - -invalid: - log_warn("fetchscript: invalid JSON"); - json_decref(doc); - - return -1; } static int -fetchscript(struct task *tk) +fetchproject(struct task *tk) { - const char *json; + json_t *doc; - if (!(json = get("fetchscript", makeurl("api/v1/script/%s", tk->job.project.name)))) + if (!(doc = get("fetchproject", makeurl("api/v1/projects/%d", tk->project_id)))) return -1; - return extract(tk, json); + return extract(tk, doc); } +/* + * Create a task to run the script. This will retrieve the project script code + * at this moment and put it in a temporary file. + */ static void createtask(struct task *tk) { if (tk->status != TASKST_PENDING) return; - log_debug("creating task (id=%lld, project=%s, tag=%s)", - tk->job.id, tk->job.project.name, tk->job.tag); - - snprintf(tk->script, sizeof (tk->script), "/tmp/sciworkerd-%s-XXXXXX", - tk->job.project.name); + log_debug("creating task (id=%d, tag=%s)", tk->job_id, tk->job_tag); + snprintf(tk->script, sizeof (tk->script), "/tmp/sciworkerd-%d-XXXXXX", tk->job_id); if ((tk->scriptfd = mkstemp(tk->script)) < 0 || fchmod(tk->scriptfd, S_IRUSR | S_IWUSR | S_IXUSR) < 0) { @@ -580,7 +514,7 @@ return; } - if (fetchscript(tk) < 0) { + if (fetchproject(tk) < 0) { unlink(tk->script); close(tk->scriptfd); tk->scriptfd = 0; @@ -588,6 +522,9 @@ spawn(tk); } +/* + * Start all pending tasks if the limit of running tasks is not reached. + */ static void startall(void) { @@ -598,9 +535,9 @@ if (tk->status == TASKST_RUNNING) ++nrunning; - if (nrunning >= (size_t)config.maxbuilds) { + if (nrunning >= (size_t)config.maxbuilds) log_debug("not spawning new process because limit is reached"); - } else { + else { tk = TAILQ_FIRST(&tasks); while (tk && nrunning++ < (size_t)config.maxbuilds) { @@ -611,6 +548,74 @@ } static void +fetchworker(void) +{ + json_t *doc; + + if (!(doc = get("fetchworker", makeurl("api/v1/workers/%s", config.worker))) || + worker_from(&worker, 1, doc) < 0) + errx(1, "unable to retrieve worker id"); + + log_info("worker id: %d", worker.id); + log_info("worker name: %s", worker.name); + log_info("worker description: %s", worker.desc); + + json_decref(doc); +} + +static void +init(void) +{ + struct sigaction sa; + + sa.sa_flags = SA_SIGINFO; + sa.sa_sigaction = complete; + sigemptyset(&sa.sa_mask); + + if (sigaction(SIGCHLD, &sa, NULL) < 0) + err(1, "sigaction"); + + log_open("sciworkerd"); + fetchworker(); + +#if 0 + if (pipe(sigpipe) < 0) + err(1, "pipe"); + if ((flags = fcntl(sigpipe[1], F_GETFL, 0)) < 0 || + fcntl(sigpipe[1], F_SETFL, flags | O_NONBLOCK) < 0) + err(1, "fcntl"); +#endif +} + +static struct fds +prepare(void) +{ + struct fds fds = {0}; + struct task *tk; + size_t i = 0; + + TAILQ_FOREACH(tk, &tasks, link) + if (tk->status == TASKST_RUNNING || tk->status == TASKST_COMPLETED) + fds.listsz++; + + fds.list = util_calloc(fds.listsz, sizeof (*fds.list)); + +#if 0 + fds.list[0].fd = sigpipe[0]; + fds.list[0].events = POLLIN; +#endif + + TAILQ_FOREACH(tk, &tasks, link) { + if (tk->status == TASKST_RUNNING || tk->status == TASKST_COMPLETED) { + fds.list[i].fd = tk->pipe[0]; + fds.list[i++].events = POLLIN | POLLPRI; + } + } + + return fds; +} + +static void run(void) { struct fds fds;
--- a/sql/init.sql Thu Jun 10 10:39:21 2021 +0200 +++ b/sql/init.sql Mon Jun 14 22:08:24 2021 +0200 @@ -18,7 +18,7 @@ project_id INTEGER NOT NULL REFERENCES project (id) ); -CREATE TABLE IF NOT EXISTS job_result( +CREATE TABLE IF NOT EXISTS jobresult( id INTEGER PRIMARY KEY AUTOINCREMENT, job_id INTEGER NOT NULL REFERENCES job (id), worker_id INTEGER NOT NULL REFERENCES worker (id),
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/sql/job-add.sql Mon Jun 14 22:08:24 2021 +0200 @@ -0,0 +1,4 @@ +INSERT INTO job( + tag, + project_id +) VALUES (?, ?)
--- a/sql/job-queue.sql Thu Jun 10 10:39:21 2021 +0200 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,4 +0,0 @@ -INSERT INTO job( - tag, - project_id -) VALUES (?, ?)
--- a/sql/job-result-todo.sql Thu Jun 10 10:39:21 2021 +0200 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,12 +0,0 @@ - SELECT job.id - , job.tag - , project.name - FROM job, project - WHERE job.project_id = project.id - AND job.id - NOT IN ( - SELECT job_result.job_id - FROM job_result - WHERE job_result.worker_id = ? - ) - LIMIT ?
--- a/sql/job-save.sql Thu Jun 10 10:39:21 2021 +0200 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,7 +0,0 @@ -INSERT INTO job_result( - job_id, - worker_id, - status, - retcode, - console -) VALUES (?, ?, ?, ?, ?)
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/sql/job-todo.sql Mon Jun 14 22:08:24 2021 +0200 @@ -0,0 +1,11 @@ + SELECT job.id + , job.tag + , job.project_id + FROM job + WHERE job.id + NOT IN ( + SELECT job_result.job_id + FROM job_result + WHERE job_result.worker_id = ? + ) + LIMIT ?
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/sql/jobresult-add.sql Mon Jun 14 22:08:24 2021 +0200 @@ -0,0 +1,7 @@ +INSERT INTO job_result( + job_id, + worker_id, + status, + retcode, + console +) VALUES (?, ?, ?, ?, ?)
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/sql/project-add.sql Mon Jun 14 22:08:24 2021 +0200 @@ -0,0 +1,6 @@ +INSERT INTO project( + name, + desc, + url, + script +) VALUES (?, ?, ?, ?)
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/sql/project-find-id.sql Mon Jun 14 22:08:24 2021 +0200 @@ -0,0 +1,4 @@ +SELECT * + FROM project + WHERE id = ? + LIMIT 1
--- a/sql/project-get.sql Thu Jun 10 10:39:21 2021 +0200 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,3 +0,0 @@ -SELECT * - FROM project - LIMIT ?
--- a/sql/project-insert.sql Thu Jun 10 10:39:21 2021 +0200 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,6 +0,0 @@ -INSERT INTO project( - name, - desc, - url, - script -) VALUES (?, ?, ?, ?)
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/sql/project-list.sql Mon Jun 14 22:08:24 2021 +0200 @@ -0,0 +1,3 @@ +SELECT * + FROM project + LIMIT ?
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/sql/worker-add.sql Mon Jun 14 22:08:24 2021 +0200 @@ -0,0 +1,4 @@ +INSERT INTO worker( + name, + desc +) VALUES (?, ?)
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/sql/worker-find-id.sql Mon Jun 14 22:08:24 2021 +0200 @@ -0,0 +1,4 @@ +SELECT * + FROM worker + WHERE id = ? + LIMIT 1
--- a/sql/worker-get.sql Thu Jun 10 10:39:21 2021 +0200 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,3 +0,0 @@ -SELECT * - FROM worker - LIMIT ?
--- a/sql/worker-insert.sql Thu Jun 10 10:39:21 2021 +0200 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,4 +0,0 @@ -INSERT INTO worker( - name, - desc -) VALUES (?, ?)
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/sql/worker-list.sql Mon Jun 14 22:08:24 2021 +0200 @@ -0,0 +1,3 @@ +SELECT * + FROM worker + LIMIT ?
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/types.c Mon Jun 14 22:08:24 2021 +0200 @@ -0,0 +1,204 @@ +#include <assert.h> +#include <errno.h> + +#include "types.h" + +typedef json_t * (*packer)(const void *); +typedef int (*unpacker)(void *, json_t *); + +static inline json_t * +job_packer(const struct job *job) +{ + return json_pack("{si si ss}", + "id", job->id, + "project_id", job->project_id, + "tag", job->tag + ); +} + +static inline int +job_unpacker(struct job *job, json_t *doc) +{ + return json_unpack(doc, "{si si ss}", + "id", &job->id, + "project_id", &job->project_id, + "tag", &job->tag + ); +} + +static inline json_t * +jobresult_packer(const struct jobresult *res) +{ + return json_pack("{si si si si ss}", + "id", res->id, + "job_id", res->job_id, + "worker_id", res->worker_id, + "exitcode", res->exitcode, + "log", res->log + ); +} + +static inline int +jobresult_unpacker(struct jobresult *res, json_t *doc) +{ + return json_unpack(doc, "{si si si si ss}", + "id", &res->id, + "job_id", &res->job_id, + "worker_id", &res->worker_id, + "exitcode", &res->exitcode, + "log", &res->log + ); +} + +static inline json_t * +worker_packer(const struct worker *w) +{ + return json_pack("{si ss ss}", + "id", w->id, + "name", w->name, + "desc", w->desc + ); +} + +static inline int +worker_unpacker(struct worker *w, json_t *doc) +{ + return json_unpack(doc, "{si ss ss}", + "id", &w->id, + "name", &w->name, + "desc", &w->desc + ); +} + +static inline json_t * +project_packer(struct project *p) +{ + return json_pack("{si ss ss ss ss}", + "id", p->id, + "name", p->name, + "desc", p->desc, + "url", p->url, + "script", p->script + ); +} + +static inline int +project_unpacker(struct project *p, json_t *doc) +{ + return json_unpack(doc, "{si ss ss ss ss}", + "id", &p->id, + "name", &p->name, + "desc", &p->desc, + "url", &p->url, + "script", &p->script + ); +} + +static json_t * +to(const void *array, size_t arraysz, size_t width, packer fn) +{ + json_t *doc; + + if (arraysz == 1) + doc = fn(array); + else { + doc = json_array(); + + for (size_t i = 0; i < arraysz; ++i) + json_array_append(doc, fn((char *)array + (i * width))); + } + + return doc; +} + +static ssize_t +from(void *array, size_t arraysz, size_t width, json_t *doc, unpacker fn) +{ + json_t *val; + size_t i, tot = 0; + + if (json_is_array(doc)) { + json_array_foreach(doc, i, val) { + if (tot >= arraysz) + return errno = ERANGE, -1; + if (fn((char *)array + (tot++ * width), val) < 0) + return errno = EILSEQ, -1; + } + } else if (json_is_object(doc)) { + tot = 1; + + if (fn(array, doc) < 0) + return errno = EILSEQ, -1; + } else + return errno = EINVAL, -1; + + return tot; +} + +json_t * +job_to(const struct job *jobs, size_t jobsz) +{ + assert(jobs); + + return to(jobs, jobsz, sizeof (*jobs), (packer)job_packer); +} + +ssize_t +job_from(struct job *jobs, size_t jobsz, json_t *doc) +{ + assert(jobs); + assert(doc); + + return from(jobs, jobsz, sizeof (*jobs), doc, (unpacker)job_unpacker); +} + +json_t * +jobresult_to(const struct jobresult *res, size_t resz) +{ + assert(res); + + return to(res, resz, sizeof (*res), (packer)jobresult_packer); +} + +ssize_t +jobresult_from(struct jobresult *res, size_t resz, json_t *doc) +{ + assert(res); + assert(doc); + + return from(res, resz, sizeof (*res), doc, (unpacker)jobresult_unpacker); +} + +json_t * +worker_to(const struct worker *w, size_t wsz) +{ + assert(w); + + return to(w, wsz, sizeof (*w), (packer)worker_packer); +} + +ssize_t +worker_from(struct worker *w, size_t wsz, json_t *doc) +{ + assert(w); + assert(doc); + + return from(w, wsz, sizeof (*w), doc, (unpacker)worker_unpacker); +} + +json_t * +project_to(const struct project *proj, size_t projsz) +{ + assert(proj); + + return to(proj, projsz, sizeof (*proj), (packer)project_packer); +} + +ssize_t +project_from(struct project *proj, size_t projsz, json_t *doc) +{ + assert(proj); + assert(doc); + + return from(proj, projsz, sizeof (*proj), doc, (unpacker)project_unpacker); +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/types.h Mon Jun 14 22:08:24 2021 +0200 @@ -0,0 +1,61 @@ +#ifndef SCI_TYPES_H +#define SCI_TYPES_H + +#include <sys/types.h> +#include <stddef.h> + +#include <jansson.h> + +struct job { + int id; + int project_id; + const char *tag; +}; + +struct jobresult { + int id; + int job_id; + int worker_id; + int exitcode; + const char *log; +}; + +struct worker { + int id; + const char *name; + const char *desc; +}; + +struct project { + int id; + const char *name; + const char *desc; + const char *url; + const char *script; +}; + +json_t * +job_to(const struct job *, size_t); + +ssize_t +job_from(struct job *, size_t, json_t *); + +json_t * +jobresult_to(const struct jobresult *, size_t); + +ssize_t +jobresult_from(struct jobresult *, size_t, json_t *); + +json_t * +worker_to(const struct worker *, size_t); + +ssize_t +worker_from(struct worker *, size_t, json_t *); + +json_t * +project_to(const struct project *, size_t); + +ssize_t +project_from(struct project *, size_t, json_t *); + +#endif /* !SCI_TYPES_H */
--- a/util.c Thu Jun 10 10:39:21 2021 +0200 +++ b/util.c Mon Jun 14 22:08:24 2021 +0200 @@ -16,18 +16,18 @@ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ +#include <sys/stat.h> #include <assert.h> #include <err.h> +#include <fcntl.h> #include <libgen.h> #include <limits.h> #include <stdarg.h> #include <stdio.h> #include <stdlib.h> #include <string.h> +#include <unistd.h> -#include <zstd.h> - -#include "base64.h" #include "util.h" void * @@ -122,28 +122,15 @@ return ret; } -size_t -util_split(char *line, const char **args, size_t max, char delim) +FILE * +util_fmemopen(void *buf, size_t size, const char *mode) { - size_t idx; - - if (!*line) - return 0; - - for (idx = 0; idx < max; ++idx) { - char *sp = strchr(line, delim); + FILE *fp; - if (!sp || idx + 1 >= max) { - args[idx++] = line; - break; - } + if (!(fp = fmemopen(buf, size, mode))) + err(1, "fmemopen"); - *sp = '\0'; - args[idx] = line; - line = sp + 1; - } - - return idx; + return fp; } char * @@ -163,67 +150,27 @@ } char * -util_zbase64_enc(const char *src) +util_read(const char *path) { - assert(src); - - char *zstd, *b64; - size_t zstdsz, b64sz, len; + int fd; + struct stat st; + char *ret; - len = strlen(src); - zstdsz = ZSTD_compressBound(len); - zstd = util_malloc(zstdsz); + if ((fd = open(path, O_RDONLY)) < 0) + return NULL; + if (fstat(fd, &st) < 0) + return close(fd), NULL; - if (ZSTD_isError(zstdsz = ZSTD_compress(zstd, zstdsz, src, len, 18))) { - free(zstd); - return NULL; + ret = util_calloc(1, st.st_size + 1); + + if (read(fd, ret, st.st_size) != st.st_size) { + free(ret); + ret = NULL; } - b64sz = B64_ENCODE_LENGTH(zstdsz); - b64 = util_calloc(1, b64sz + 1); - b64_encode(zstd, zstdsz, b64, b64sz); - free(zstd); - - return b64; -} - -char * -util_zbase64_dec(const char *src) -{ - assert(src); - - char *zstd, *text; - size_t zstdsz, textsz, len; - - len = strlen(src); - zstdsz = B64_DECODE_LENGTH(len) + 1; - zstd = util_calloc(1, zstdsz); + close(fd); - if ((zstdsz = b64_decode(src, len, zstd, zstdsz)) == (size_t)-1) { - free(zstd); - return NULL; - } - - switch ((textsz = ZSTD_getFrameContentSize(zstd, zstdsz))) { - case ZSTD_CONTENTSIZE_UNKNOWN: - case ZSTD_CONTENTSIZE_ERROR: - free(zstd); - return NULL; - default: - break; - } - - text = util_calloc(1, textsz + 1); - - if (ZSTD_isError((textsz = ZSTD_decompress(text, textsz, zstd, zstdsz)))) { - free(zstd); - free(text); - return NULL; - } - - free(zstd); - - return text; + return ret; } const char *
--- a/util.h Thu Jun 10 10:39:21 2021 +0200 +++ b/util.h Mon Jun 14 22:08:24 2021 +0200 @@ -20,6 +20,7 @@ #define SCI_UTIL_H #include <stddef.h> +#include <stdio.h> #define UTIL_SIZE(x) (sizeof (x) / sizeof (x[0])) @@ -50,17 +51,14 @@ char * util_dirname(const char *); -size_t -util_split(char *, const char **, size_t, char); +FILE * +util_fmemopen(void *, size_t, const char *); char * util_printf(char *, size_t, const char *, ...); char * -util_zbase64_enc(const char *); - -char * -util_zbase64_dec(const char *); +util_read(const char *); const char * util_path(const char *);
--- a/worker.h Thu Jun 10 10:39:21 2021 +0200 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,15 +0,0 @@ -#ifndef SCI_WORKER_H -#define SCI_WORKER_H - -#include <stdint.h> - -#define WORKER_NAME_MAX 32 -#define WORKER_DESC_MAX 256 - -struct worker { - int64_t id; - char name[WORKER_NAME_MAX]; - char desc[WORKER_DESC_MAX]; -}; - -#endif /* !SCI_WORKER_H */