Mercurial > sci
changeset 1:5afdb14df924
sci: add support for storing results
author | David Demelier <markand@malikania.fr> |
---|---|
date | Tue, 08 Jun 2021 08:40:01 +0200 |
parents | f1de39079243 |
children | 5fa3d2f479b2 |
files | LICENSE.md Makefile base64.c base64.h config.def.h db.c db.h job.c job.h req.c req.h scictl.c scid.c sciworkerd.c sql/init.sql sql/job-queue-list.sql sql/job-save.sql sql/worker-find.sql util.c util.h |
diffstat | 20 files changed, 816 insertions(+), 85 deletions(-) [+] |
line wrap: on
line diff
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/LICENSE.md Tue Jun 08 08:40:01 2021 +0200 @@ -0,0 +1,16 @@ +sci ISC LICENSE +=============== + +Copyright (c) 2021 David Demelier <markand@malikania.fr> + +Permission to use, copy, modify, and/or distribute this software for any +purpose with or without fee is hereby granted, provided that the above +copyright notice and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
--- a/Makefile Mon Jun 07 09:41:37 2021 +0200 +++ b/Makefile Tue Jun 08 08:40:01 2021 +0200 @@ -1,33 +1,54 @@ +# +# Makefile -- POSIX Makefile for sci +# +# Copyright (c) 2021 David Demelier <markand@malikania.fr> +# +# Permission to use, copy, modify, and/or distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +# + .POSIX: include config.mk -SCID_SRCS= db.c \ - job.c \ - util.c \ - scid.c \ - log.c \ +SCID_SRCS= base64.c \ + db.c \ + job.c \ + util.c \ + scid.c \ + log.c \ extern/libsqlite/sqlite3.c -SCID_DATA= sql/init.h \ - sql/job-queue.h \ - sql/job-queue-list.h \ - sql/project-find.h \ - sql/project-get.h \ - sql/project-insert.h \ - sql/worker-get.h \ +SCID_DATA= sql/init.h \ + sql/job-queue.h \ + sql/job-queue-list.h \ + sql/job-save.h \ + sql/project-find.h \ + sql/project-get.h \ + sql/project-insert.h \ + sql/worker-get.h \ + sql/worker-find.h \ sql/worker-insert.h SCID_OBJS= ${SCID_SRCS:.c=.o} SCID_DEPS= ${SCID_SRCS:.c=.d} -SCIWORKERD_SRCS= sciworkerd.c util.c log.c +SCIWORKERD_SRCS= base64.c sciworkerd.c util.c log.c SCIWORKERD_OBJS= ${SCIWORKERD_SRCS:.c=.o} SCIWORKERD_DEPS= ${SCIWORKERD_SRCS:.c=.d} -SCICTL_SRCS= req.c scictl.c +SCICTL_SRCS= base64.c req.c scictl.c util.c SCICTL_OBJS= ${SCICTL_SRCS:.c=.o} SCICTL_DEPS= ${SCICTL_SRCS:.c=.d} -SCIWEBD_SRCS= sciwebd.c +SCIWEBD_SRCS= base64.c sciwebd.c util.c SCIWEBD_OBJS= ${SCIWEBD_SRCS:.c=.o} SCIWEBD_DEPS= ${SCIWEBD_SRCS:.c=.d} @@ -45,6 +66,9 @@ KCGI_INCS= `pkg-config --cflags kcgi` KCGI_LIBS= `pkg-config --libs kcgi` +ZSTD_INCS= `pkg-config --cflags libzstd` +ZSTD_LIBS= `pkg-config --libs libzstd` + DEFS= -DVARDIR=\"${VARDIR}\" \ -DTMPDIR=\"${TMPDIR}\" @@ -54,7 +78,7 @@ all: scid scictl sciworkerd sciwebd .c.o: - ${CC} ${DEFS} ${LIBBSD_INCS} ${KCGI_INCS} ${CFLAGS} -c $< -o $@ + ${CC} ${DEFS} ${LIBBSD_INCS} ${KCGI_INCS} ${ZSTD_INCS} ${CFLAGS} -c $< -o $@ .sql.h: ./bcc -sc0 $< $< > $@ @@ -72,25 +96,25 @@ ${SCID_OBJS}: config.h ${SCID_DATA} scid: ${SCID_OBJS} - ${CC} ${CFLAGS} -o $@ ${SCID_OBJS} ${LIBBSD_LIBS} ${LDFLAGS} + ${CC} ${CFLAGS} -o $@ ${SCID_OBJS} ${LIBBSD_LIBS} ${ZSTD_LIBS} ${LDFLAGS} ${SCIWORKERD_OBJS}: config.h sciworkerd: ${SCIWORKERD_OBJS} - ${CC} ${CFLAGS} -o $@ ${SCIWORKERD_OBJS} ${LIBBSD_LIBS} ${LIBCURL_LIBS} ${LDFLAGS} + ${CC} ${CFLAGS} -o $@ ${SCIWORKERD_OBJS} ${LIBBSD_LIBS} ${LIBCURL_LIBS} ${ZSTD_LIBS} ${LDFLAGS} ${SCICTL_OBJS}: config.h scictl: ${SCICTL_OBJS} - ${CC} ${CFLAGS} -o $@ ${SCICTL_OBJS} ${LIBBSD_LIBS} ${LDFLAGS} + ${CC} ${CFLAGS} -o $@ ${SCICTL_OBJS} ${LIBBSD_LIBS} ${ZSTD_LIBS} ${LDFLAGS} ${SCIWEBD_OBJS}: config.h sciwebd: ${SCIWEBD_OBJS} - ${CC} ${CFLAGS} -o $@ ${SCIWEBD_OBJS} ${LIBBSD_LIBS} ${KCGI_LIBS} ${LDFLAGS} + ${CC} ${CFLAGS} -o $@ ${SCIWEBD_OBJS} ${LIBBSD_LIBS} ${KCGI_LIBS} ${ZSTD_LIBS} ${LDFLAGS} clean: - rm -f bcc + rm -f bcc config.h rm -f scid ${SCID_OBJS} ${SCID_DEPS} ${SCID_DATA} rm -f scictl ${SCICTL_OBJS} ${SCICTL_DEPS} rm -f sciworkerd ${SCIWORKERD_OBJS} ${SCIWORKERD_DEPS}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/base64.c Tue Jun 08 08:40:01 2021 +0200 @@ -0,0 +1,186 @@ +/* + * base64.h -- base64 encoding and decoding + * + * Copyright (c) 2013-2021 David Demelier <markand@malikania.fr> + * + * Permission to use, copy, modify, and/or distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ + +#include <assert.h> +#include <ctype.h> +#include <errno.h> +#include <string.h> + +#include "base64.h" + +int +b64_isbase64(unsigned char ch) +{ + return isalnum(ch) || ch == '+' || ch == '-' || ch == '_' || ch == '/'; +} + +int +b64_isvalid(unsigned char ch) +{ + return b64_isbase64(ch) || ch == '='; +} + +unsigned char +b64_lookup(unsigned char value) +{ + assert(value < 64); + + static const char *table = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + + return table[value]; +} + +unsigned char +b64_rlookup(unsigned char ch) +{ + assert(b64_isbase64(ch)); + + if (ch >= '0' && ch <= '9') + return ch + 4; + if (ch >= 'A' && ch <= 'Z') + return ch - 65; + if (ch >= 'a' && ch <= 'z') + return ch - 71; + + /* '-' is base64url support. */ + return ch == '+' || ch == '-' ? 62U : 63U; +} + +size_t +b64_encode(const char *src, size_t srcsz, char *dst, size_t dstsz) +{ + assert(src); + assert(dst); + + size_t nwritten = 0; + + if (srcsz == (size_t)-1) + srcsz = strlen(src); + + while (srcsz && dstsz) { + char inputbuf[3] = {0}; + int count = 0; + + while (srcsz && count < 3) { + inputbuf[count++] = *src++; + --srcsz; + } + + if (dstsz < 4) { + errno = ERANGE; + return -1; + } + + *dst++ = b64_lookup(inputbuf[0] >> 2 & 0x3f); + *dst++ = b64_lookup((inputbuf[0] << 4 & 0x3f) | (inputbuf[1] >> 4 & 0x0f)); + + if (count < 2) + *dst++ = '='; + else + *dst++ = b64_lookup((inputbuf[1] << 2 & 0x3c) | (inputbuf[2] >> 6 & 0x03)); + + if (count < 3) + *dst++ = '='; + else + *dst++ = b64_lookup(inputbuf[2] & 0x3f); + + nwritten += 4; + dstsz -= 4; + } + + /* Not enough room to store '\0'. */ + if (dstsz == 0) { + errno = ERANGE; + return -1; + } + + *dst = '\0'; + + return nwritten; +} + +size_t +b64_decode(const char *src, size_t srcsz, char *dst, size_t dstsz) +{ + assert(src); + assert(dst); + + size_t nwritten = 0; + + if (srcsz == (size_t)-1) + srcsz = strlen(src); + + while (srcsz && dstsz) { + int i = 0, r = 3; + unsigned int inputbuf[4] = {0}; + + for (; srcsz && i < 4; i++) { + if (*src == '=') { + /* + * '=' is only allowed in last 2 characters, + * otherwise it means we need less data. + */ + if (i <= 1) + goto eilseq; + + /* Less data required. */ + --r; + } else if (!b64_isvalid(*src)) + goto eilseq; + + if (b64_isbase64(*src)) + inputbuf[i] = b64_rlookup(*src); + + ++src; + --srcsz; + } + + /* Make sure we haven't seen AB=Z as well. */ + if (i != 4 || (src[-2] == '=' && src[-1] != '=')) + goto eilseq; + if ((size_t)r >= dstsz) + goto erange; + + *dst++ = ((inputbuf[0] << 2) & 0xfc) | + ((inputbuf[1] >> 4) & 0x03); + + if (r >= 2) + *dst++ = ((inputbuf[1] << 4) & 0xf0) | ((inputbuf[2] >> 2) & 0x0f); + if (r >= 3) + *dst++ = ((inputbuf[2] << 6) & 0xc0) | (inputbuf[3] & 0x3f); + + nwritten += r; + dstsz -= r; + } + + /* Not enough room to store '\0'. */ + if (dstsz == 0) + goto erange; + + *dst = '\0'; + + return nwritten; + +eilseq: + errno = EILSEQ; + return -1; + +erange: + errno = ERANGE; + return -1; +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/base64.h Tue Jun 08 08:40:01 2021 +0200 @@ -0,0 +1,53 @@ +/* + * base64.h -- base64 encoding and decoding + * + * Copyright (c) 2013-2021 David Demelier <markand@malikania.fr> + * + * Permission to use, copy, modify, and/or distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ + +#ifndef BASE64_H +#define BASE64_H + +#include <stddef.h> + +#if defined(__cplusplus) +extern "C" { +#endif + +#define B64_ENCODE_LENGTH(x) (4 * ((x) / 3 + 1)) +#define B64_DECODE_LENGTH(x) (3 * ((x) / 4)) + +int +b64_isbase64(unsigned char ch); + +int +b64_isvalid(unsigned char ch); + +unsigned char +b64_lookup(unsigned char value); + +unsigned char +b64_rlookup(unsigned char ch); + +size_t +b64_encode(const char *src, size_t srcsz, char *dst, size_t dstsz); + +size_t +b64_decode(const char *src, size_t srcsz, char *dst, size_t dstsz); + +#if defined(__cplusplus) +} +#endif + +#endif /* !BASE64_H */
--- a/config.def.h Mon Jun 07 09:41:37 2021 +0200 +++ b/config.def.h Tue Jun 08 08:40:01 2021 +0200 @@ -1,10 +1,33 @@ -#ifndef CONFIG_H -#define CONFIG_H - -#define JOB_OUT_MAX 10000000 /* Maximum job out/err log size in bytes. */ +/* + * config.def.h -- configuration and limits + * + * Copyright (c) 2021 David Demelier <markand@malikania.fr> + * + * Permission to use, copy, modify, and/or distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ -/* Project limits. */ -#define PROJECT_MAX 64 -#define WORKER_MAX 32 +#ifndef SCI_CONFIG_H +#define SCI_CONFIG_H + +/* I/O limits */ +#define SCI_CONSOLE_MAX 1048576 /* Build log max */ +#define SCI_MSG_MAX (SCI_CONSOLE_MAX + 1024) /* Network message max. */ -#endif /* !CONFIG_H */ +/* Database limits. */ +#define SCI_PROJECT_MAX 64 /* Projects allowed in database. */ +#define SCI_WORKER_MAX 32 /* Workers allowed in database. */ + +/* Usage limits. */ +#define SCI_JOB_LIST_MAX 128 /* Jobs max list size. */ + +#endif /* !SCI_CONFIG_H */
--- a/db.c Mon Jun 07 09:41:37 2021 +0200 +++ b/db.c Tue Jun 08 08:40:01 2021 +0200 @@ -13,10 +13,12 @@ #include "sql/init.h" #include "sql/job-queue.h" #include "sql/job-queue-list.h" +#include "sql/job-save.h" #include "sql/project-insert.h" #include "sql/project-get.h" #include "sql/project-find.h" #include "sql/worker-get.h" +#include "sql/worker-find.h" #include "sql/worker-insert.h" #define CHAR(v) (const char *)(v) @@ -93,7 +95,7 @@ ssize_t ret = 0; if (sqlite3_prepare(db, CHAR(sql_project_get), -1, &stmt, NULL) != SQLITE_OK) { - log_warn("%s", sqlite3_errmsg(db)); + log_warn("db: %s", sqlite3_errmsg(db)); return -1; } @@ -129,7 +131,7 @@ sqlite3_err: if (ret < 0) - log_warn("%s", sqlite3_errmsg(db)); + log_warn("db: %s", sqlite3_errmsg(db)); if (stmt) sqlite3_finalize(stmt); @@ -185,7 +187,38 @@ sqlite3_err: if (ret < 0) - log_warn("%s", sqlite3_errmsg(db)); + log_warn("db: %s", sqlite3_errmsg(db)); + if (stmt) + sqlite3_finalize(stmt); + + return ret; +} + +int +db_worker_find(struct worker *w, const char *name) +{ + assert(w); + assert(name); + + sqlite3_stmt *stmt = NULL; + int ret = -1; + + if (sqlite3_prepare(db, CHAR(sql_worker_find), -1, &stmt, NULL) != SQLITE_OK) + goto sqlite3_err; + + sqlite3_bind_text(stmt, 1, name, -1, SQLITE_STATIC); + + if (sqlite3_step(stmt) != SQLITE_ROW) + goto sqlite3_err; + + ret = 0; + w->id = sqlite3_column_int64(stmt, 0); + strlcpy(w->name, CHAR(sqlite3_column_text(stmt, 1)), sizeof (w->name)); + strlcpy(w->desc, CHAR(sqlite3_column_text(stmt, 2)), sizeof (w->desc)); + +sqlite3_err: + if (ret < 0) + log_warn("db: %s", sqlite3_errmsg(db)); if (stmt) sqlite3_finalize(stmt); @@ -204,7 +237,7 @@ goto sqlite3_err; sqlite3_bind_text(stmt, 1, job->tag, -1, SQLITE_STATIC); - sqlite3_bind_int64(stmt, 2, job->project_id); + sqlite3_bind_int64(stmt, 2, job->project.id); if (sqlite3_step(stmt) != SQLITE_DONE) goto sqlite3_err; @@ -214,7 +247,7 @@ sqlite3_err: if (ret < 0) - log_warn("%s", sqlite3_errmsg(db)); + log_warn("db: %s", sqlite3_errmsg(db)); if (stmt) sqlite3_finalize(stmt); @@ -222,28 +255,72 @@ } ssize_t -db_job_queue_list(struct job *jobs, size_t jobsz, int64_t project_id) +db_job_result_todo(struct job_result *re, size_t resz, int64_t project_id) { - assert(jobs); + assert(re); sqlite3_stmt *stmt = NULL; ssize_t ret = 0; - struct job *j = jobs; if (sqlite3_prepare(db, CHAR(sql_job_queue_list), -1, &stmt, NULL) != SQLITE_OK) { - log_warn("%s", sqlite3_errmsg(db)); + log_warn("db: %s", sqlite3_errmsg(db)); return -1; } sqlite3_bind_int64(stmt, 1, project_id); - sqlite3_bind_int64(stmt, 1, jobsz); + sqlite3_bind_int64(stmt, 2, resz); + + while (sqlite3_step(stmt) == SQLITE_ROW && (size_t)ret++ < resz) { + memset(re, 0, sizeof (*re)); + + re->job.id = sqlite3_column_int64(stmt, 0); + strlcpy(re->job.tag, CHAR(sqlite3_column_text(stmt, 1)), sizeof (re->job.tag)); + + re->worker.id = sqlite3_column_int64(stmt, 2); + strlcpy(re->worker.name, CHAR(sqlite3_column_text(stmt, 3)), sizeof (re->worker.name)); + strlcpy(re->worker.desc, CHAR(sqlite3_column_text(stmt, 4)), sizeof (re->worker.desc)); + + re->job.project.id = sqlite3_column_int64(stmt, 5); + strlcpy(re->job.project.name, CHAR(sqlite3_column_text(stmt, 6)), sizeof (re->job.project.name)); + strlcpy(re->job.project.desc, CHAR(sqlite3_column_text(stmt, 7)), sizeof (re->job.project.desc)); + strlcpy(re->job.project.url, CHAR(sqlite3_column_text(stmt, 8)), sizeof (re->job.project.url)); + strlcpy(re->job.project.script, CHAR(sqlite3_column_text(stmt, 9)), sizeof (re->job.project.script)); + + ++re; + }; + + if (stmt) + sqlite3_finalize(stmt); + + return ret; +} - for (; sqlite3_step(stmt) == SQLITE_ROW && (size_t)ret < jobsz; ++ret, ++j) { - j->id = sqlite3_column_int64(stmt, 0); - j->project_id = sqlite3_column_int64(stmt, 2); - strlcpy(j->tag, CHAR(sqlite3_column_text(stmt, 1)), sizeof (j->tag)); - } +int +db_job_save(struct job_result *r) +{ + assert(r); + + sqlite3_stmt *stmt = NULL; + int ret = -1; + + if (sqlite3_prepare(db, CHAR(sql_job_save), -1, &stmt, NULL) != SQLITE_OK) + goto sqlite3_err; + sqlite3_bind_int64(stmt, 1, r->job.id); + sqlite3_bind_int64(stmt, 2, r->worker.id); + sqlite3_bind_int(stmt, 3, r->status); + sqlite3_bind_int(stmt, 4, r->retcode); + sqlite3_bind_text(stmt, 5, r->console, -1, SQLITE_STATIC); + + if (sqlite3_step(stmt) != SQLITE_DONE) + goto sqlite3_err; + + ret = 0; + r->id = sqlite3_last_insert_rowid(db); + +sqlite3_err: + if (ret < 0) + log_warn("db: %s", sqlite3_errmsg(db)); if (stmt) sqlite3_finalize(stmt);
--- a/db.h Mon Jun 07 09:41:37 2021 +0200 +++ b/db.h Tue Jun 08 08:40:01 2021 +0200 @@ -8,6 +8,7 @@ struct project; struct worker; struct job; +struct job_result; int db_open(const char *); @@ -28,10 +29,16 @@ db_worker_get(struct worker *, size_t); int +db_worker_find(struct worker *, const char *); + +int db_job_queue(struct job *); ssize_t -db_job_queue_list(struct job *, size_t, int64_t); +db_job_result_todo(struct job_result *, size_t, int64_t); + +int +db_job_save(struct job_result *); void db_finish(void);
--- a/job.c Mon Jun 07 09:41:37 2021 +0200 +++ b/job.c Tue Jun 08 08:40:01 2021 +0200 @@ -1,3 +1,21 @@ +/* + * job.c -- job description and result + * + * Copyright (c) 2021 David Demelier <markand@malikania.fr> + * + * Permission to use, copy, modify, and/or distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ + #include <assert.h> #include <stdlib.h> #include <string.h>
--- a/job.h Mon Jun 07 09:41:37 2021 +0200 +++ b/job.h Tue Jun 08 08:40:01 2021 +0200 @@ -1,5 +1,5 @@ /* - * job.h -- main scid(8) program file + * job.h -- job description and result * * Copyright (c) 2021 David Demelier <markand@malikania.fr> * @@ -23,15 +23,27 @@ #define JOB_TAG_MAX 128 +#include "project.h" +#include "worker.h" + +enum job_status { + JOB_STATUS_TODO, + JOB_STATUS_SUCCESS, + JOB_STATUS_FAIL +}; + struct job { int64_t id; - int64_t project_id; + struct project project; char tag[JOB_TAG_MAX]; }; struct job_result { int64_t id; - int status; + struct job job; + struct worker worker; + enum job_status status; + int retcode; char *console; };
--- a/req.c Mon Jun 07 09:41:37 2021 +0200 +++ b/req.c Tue Jun 08 08:40:01 2021 +0200 @@ -3,14 +3,18 @@ #include <sys/time.h> #include <sys/un.h> #include <assert.h> +#include <err.h> #include <errno.h> #include <stdarg.h> #include <stdio.h> +#include <stdlib.h> #include <string.h> #include <unistd.h> +#include "job.h" +#include "project.h" #include "req.h" -#include "project.h" +#include "util.h" #include "worker.h" static int sock; @@ -68,6 +72,38 @@ return res; } +static char * +readfile(const char *path) +{ + FILE *fp, *str; + char *console, *b64, buf[BUFSIZ]; + size_t consolesz, nr; + + if (strcmp(path, "-") == 0) + fp = stdin; + else if (!(fp = fopen(path, "r"))) + return NULL; + + if (!(str = open_memstream(&console, &consolesz))) + err(1, NULL); + + while ((nr = fread(buf, 1, sizeof (buf), fp))) + fwrite(buf, 1, nr, str); + + if ((ferror(fp) && !feof(fp)) || (ferror(str) && !feof(fp))) { + free(console); + console = NULL; + } + + fclose(str); + fclose(fp); + + b64 = util_zbase64_enc(console); + free(console); + + return b64; +} + int req_connect(const char *path) { @@ -101,6 +137,62 @@ } struct req +req_job_list(struct job_result *jobs, size_t *jobsz, const char *project) +{ + assert(jobs); + assert(jobsz); + + struct req req; + char fmt[128], *token, *p = req.msg; + size_t tot = 0; + + if ((req = ask("job-list %s", project)).status) + return req; + + snprintf(fmt, sizeof (fmt), "%%zd|%%%zu[^|]|%%%zu[^|]|%%%zu[^\n]\n", + sizeof (jobs->job.tag), sizeof (jobs->job.project.name), + sizeof (jobs->worker.name)); + + while ((token = strtok_r(p, "\n", &p)) && tot < *jobsz) { + if (sscanf(token, fmt, &jobs->job.id, jobs->job.tag, + jobs->job.project.name, jobs->worker.name) == 4) { + ++jobs; + ++tot; + } + } + + *jobsz = tot; + + return req; +} + +struct req +req_job_save(const char *id, + const char *worker, + const char *status, + const char *retcode, + const char *console) +{ + assert(id); + assert(worker); + assert(status); + assert(retcode); + assert(console); + + char *b64; + struct req req = {0}; + + if (!(b64 = readfile(console))) + req.status = errno; + else { + req = ask("job-save %s|%s|%s|%s|%s", id, worker, status, retcode, b64); + free(b64); + } + + return req; +} + +struct req req_project_add(const struct project *p) { assert(p);
--- a/req.h Mon Jun 07 09:41:37 2021 +0200 +++ b/req.h Tue Jun 08 08:40:01 2021 +0200 @@ -12,6 +12,7 @@ struct worker; struct project; +struct job_result; int req_connect(const char *); @@ -20,6 +21,16 @@ req_job_queue(const char *, const char *); struct req +req_job_list(struct job_result *, size_t *, const char *); + +struct req +req_job_save(const char *, + const char *, + const char *, + const char *, + const char *); + +struct req req_project_add(const struct project *); struct req
--- a/scictl.c Mon Jun 07 09:41:37 2021 +0200 +++ b/scictl.c Tue Jun 08 08:40:01 2021 +0200 @@ -8,6 +8,7 @@ #include "config.h" #include "project.h" +#include "job.h" #include "req.h" #include "util.h" #include "worker.h" @@ -22,11 +23,13 @@ noreturn static void help(void) { - fprintf(stderr, "usage: job-queue project tag\n"); - fprintf(stderr, " project-add name desc url script\n"); - fprintf(stderr, " project-list\n"); - fprintf(stderr, " worker-add name desc\n"); - fprintf(stderr, " worker-list\n"); + fprintf(stderr, "usage: %s job-queue project tag\n", getprogname()); + fprintf(stderr, " %s job-list project\n", getprogname()); + fprintf(stderr, " %s job-save id worker status retcode console\n", getprogname()); + fprintf(stderr, " %s project-add name desc url script\n", getprogname()); + fprintf(stderr, " %s project-list\n", getprogname()); + fprintf(stderr, " %s worker-add name desc\n", getprogname()); + fprintf(stderr, " %s worker-list\n", getprogname()); exit(0); } @@ -40,6 +43,38 @@ } static struct req +cmd_job_list(int argc, char **argv) +{ + struct job_result jobs[SCI_JOB_LIST_MAX]; + size_t jobsz = UTIL_SIZE(jobs); + struct req req; + + if (argc < 1) + usage(); + + if ((req = req_job_list(jobs, &jobsz, argv[0])).status) + return req; + + printf("%-16s%-16s%-16s%s\n", "ID", "TAG", "PROJECT", "WORKER"); + + for (size_t i = 0; i < jobsz; ++i) { + printf("%-16lld%-16s%-16s%s\n", (long long int)jobs[i].job.id, + jobs[i].job.tag, jobs[i].job.project.name, jobs[i].worker.name); + } + + return req; +} + +static struct req +cmd_job_save(int argc, char **argv) +{ + if (argc < 5) + usage(); + + return req_job_save(argv[0], argv[1], argv[2], argv[3], argv[4]); +} + +static struct req cmd_project_add(int argc, char **argv) { struct project pc; @@ -71,15 +106,11 @@ if ((req = req_project_list(pc, &pcsz)).status) return req; - for (size_t i = 0; i < pcsz; ++i) { - printf("%-16s%s\n", "name:", pc[i].name); - printf("%-16s%s\n", "description:", pc[i].desc); - printf("%-16s%s\n", "url:", pc[i].url); - printf("%-16s%s\n", "script:", pc[i].script); + printf("%-16s%-24s%-20s%s\n", "NAME", "DESCRIPTION", "URL", "SCRIPT"); - if (i + 1 < pcsz) - printf("\n"); - } + for (size_t i = 0; i < pcsz; ++i) + printf("%-16s%-24s%-20s%s\n", pc[i].name, pc[i].desc, + pc[i].url, pc[i].script); return req; } @@ -112,13 +143,10 @@ if ((req = req_worker_list(wk, &wksz)).status) return req; - for (size_t i = 0; i < wksz; ++i) { - printf("%-16s%s\n", "name:", wk[i].name); - printf("%-16s%s\n", "description:", wk[i].desc); + printf("%-16s%s\n", "NAME", "DESCRIPTION"); - if (i + 1 < wksz) - printf("\n"); - } + for (size_t i = 0; i < wksz; ++i) + printf("%-16s%s\n", wk[i].name, wk[i].desc); return req; } @@ -128,6 +156,8 @@ struct req (*exec)(int, char **); } commands[] = { { "job-queue", cmd_job_queue }, + { "job-list", cmd_job_list }, + { "job-save", cmd_job_save }, { "project-add", cmd_project_add }, { "project-list", cmd_project_list }, { "worker-add", cmd_worker_add }, @@ -139,7 +169,7 @@ main(int argc, char **argv) { const char *sock = VARDIR "/run/sci.sock"; - int ch; + int ch, cmdfound = 0; setprogname("scictl"); @@ -168,6 +198,7 @@ if (strcmp(commands[i].name, argv[0]) == 0) { res = commands[i].exec(--argc, ++argv); + cmdfound = 1; if (res.status) warnx("%s", res.msg); @@ -176,5 +207,8 @@ } } + if (!cmdfound) + errx(1, "abort: command %s not found", argv[0]); + req_finish(); }
--- a/scid.c Mon Jun 07 09:41:37 2021 +0200 +++ b/scid.c Tue Jun 08 08:40:01 2021 +0200 @@ -33,6 +33,7 @@ #include "job.h" #include "log.h" #include "project.h" +#include "util.h" #include "worker.h" static char dbpath[PATH_MAX] = VARDIR "/db/sci/sci.db"; @@ -141,7 +142,7 @@ return ENOENT; } - job.project_id = project.id; + job.project.id = project.id; strlcpy(job.tag, args[1], sizeof (job.tag)); if (db_job_queue(&job) < 0) @@ -153,6 +154,86 @@ return ok(fd); } +static int +cmd_job_list(int fd, char *cmd) +{ + char *args[1] = {0}, buf[SCI_MSG_MAX]; + struct job_result jobs[SCI_JOB_LIST_MAX]; + struct project project; + ssize_t n; + FILE *fp; + + if (split(cmd, args, 1) != 1) { + log_warn("invalid job-list invocation"); + return EINVAL; + } + if (!(fp = fmemopen(buf, sizeof (buf), "w"))) + return ENOMEM; + + strlcpy(project.name, args[0], sizeof (project.name)); + + if (db_project_find(&project) < 0) { + log_warn("project %s not found", args[0]); + return ENOENT; + } + + if ((n = db_job_result_todo(jobs, UTIL_SIZE(jobs), project.id)) < 0) + return answer(fd, "ERR unable to retrieve jobs list"); + + fprintf(fp, "OK\n"); + + for (ssize_t i = 0; i < n; ++i) + fprintf(fp, "%lld|%s|%s|%s\n", (long long int)jobs[i].job.id, + jobs[i].job.tag, jobs[i].job.project.name, jobs[i].worker.name); + + fclose(fp); + + return answer(fd, "%s", buf); +} + +/* + * Request + * ------- + * + * job-save id|worker|status|retcode|console + */ +static int +cmd_job_save(int fd, char *cmd) +{ + char *args[5] = {0}; + struct job_result res; + + if (split(cmd, args, 5) != 5) { + log_warn("invalid job-save invocation"); + return EINVAL; + } + if (db_worker_find(&res.worker, args[1]) < 0) { + log_warn("worker %s not found", args[1]); + return ENOENT; + } + + res.id = strtoll(args[0], NULL, 10); + res.status = strtoll(args[2], NULL, 10); + res.retcode = strtoll(args[3], NULL, 10); + res.console = util_zbase64_dec(args[4]); + + if (!res.console) { + log_warn("failed to decode console data"); + return EINVAL; + } + + if (db_job_save(&res) < 0) { + log_warn("failed to save job result"); + return EINVAL; + } + + log_info("save job info (%lld, status=%d retcode=%d)", + (long long int)res.id, res.status, res.retcode); + free(res.console); + + return ok(fd); +} + /* * Request * ------- @@ -210,7 +291,7 @@ { (void)cmd; - char buf[1024]; + char buf[SCI_MSG_MAX]; struct project projects[SCI_PROJECT_MAX]; ssize_t np; FILE *fp; @@ -285,7 +366,7 @@ { (void)cmd; - char buf[1024]; + char buf[SCI_MSG_MAX]; struct worker wk[SCI_WORKER_MAX]; ssize_t np; FILE *fp; @@ -315,6 +396,8 @@ int (*exec)(int, char *); } cmds[] = { { "job-queue", cmd_job_queue }, + { "job-list", cmd_job_list }, + { "job-save", cmd_job_save }, { "project-add", cmd_project_add }, { "project-list", cmd_project_list }, { "worker-add", cmd_worker_add },
--- a/sciworkerd.c Mon Jun 07 09:41:37 2021 +0200 +++ b/sciworkerd.c Tue Jun 08 08:40:01 2021 +0200 @@ -22,7 +22,7 @@ int running; int pipe[2]; char project[PROJECT_NAME_MAX]; - char out[SCI_JOB_OUTPUT_MAX]; + char out[SCI_CONSOLE_MAX]; TAILQ_ENTRY(job) link; };
--- a/sql/init.sql Mon Jun 07 09:41:37 2021 +0200 +++ b/sql/init.sql Tue Jun 08 08:40:01 2021 +0200 @@ -22,6 +22,7 @@ id INTEGER PRIMARY KEY AUTOINCREMENT, job_id INTEGER NOT NULL REFERENCES job (id), worker_id INTEGER NOT NULL REFERENCES worker (id), - status INTEGER NOT NULL, - console TEXT NOT NULL + status INTEGER DEFAULT 0, + retcode INTEGER DEFAULT 0, + console TEXT DEFAULT NULL );
--- a/sql/job-queue-list.sql Mon Jun 07 09:41:37 2021 +0200 +++ b/sql/job-queue-list.sql Tue Jun 08 08:40:01 2021 +0200 @@ -1,10 +1,20 @@ -SELECT * - FROM job - WHERE job.id - AND project_id = ? -NOT IN ( - SELECT job_result.job_id - FROM job_result - WHERE worker_id = ? -) - LIMIT ? + SELECT job.id + , job.tag + , worker.id + , worker.name + , worker.desc + , project.id + , project.name + , project.desc + , project.url + , project.script + FROM job, worker, project + WHERE job.project_id = ? + AND job.project_id = project.id + AND job.id + NOT IN ( + SELECT job_result.job_id + FROM job_result + WHERE job_result.worker_id = worker.id + ) + LIMIT ?
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/sql/job-save.sql Tue Jun 08 08:40:01 2021 +0200 @@ -0,0 +1,7 @@ +INSERT INTO job_result( + job_id, + worker_id, + status, + retcode, + console +) VALUES (?, ?, ?, ?, ?)
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/sql/worker-find.sql Tue Jun 08 08:40:01 2021 +0200 @@ -0,0 +1,4 @@ +SELECT * + FROM worker + WHERE name = ? + LIMIT 1
--- a/util.c Mon Jun 07 09:41:37 2021 +0200 +++ b/util.c Tue Jun 08 08:40:01 2021 +0200 @@ -25,6 +25,9 @@ #include <stdlib.h> #include <string.h> +#include <zstd.h> + +#include "base64.h" #include "util.h" void * @@ -158,3 +161,67 @@ return buf; } + +char * +util_zbase64_enc(const char *src) +{ + assert(src); + + char *zstd, *b64; + size_t zstdsz, b64sz, len; + + len = strlen(src); + zstdsz = ZSTD_compressBound(len); + zstd = util_malloc(zstdsz); + + if (ZSTD_isError(zstdsz = ZSTD_compress(zstd, zstdsz, src, len, 18))) { + free(zstd); + return NULL; + } + + b64sz = B64_ENCODE_LENGTH(zstdsz); + b64 = util_calloc(1, b64sz + 1); + b64_encode(zstd, zstdsz, b64, b64sz); + free(zstd); + + return b64; +} + +char * +util_zbase64_dec(const char *src) +{ + assert(src); + + char *zstd, *text; + size_t zstdsz, textsz, len; + + len = strlen(src); + zstdsz = B64_DECODE_LENGTH(len) + 1; + zstd = util_calloc(1, zstdsz); + + if ((zstdsz = b64_decode(src, len, zstd, zstdsz)) == (size_t)-1) { + free(zstd); + return NULL; + } + + switch ((textsz = ZSTD_getFrameContentSize(zstd, zstdsz))) { + case ZSTD_CONTENTSIZE_UNKNOWN: + case ZSTD_CONTENTSIZE_ERROR: + free(zstd); + return NULL; + default: + break; + } + + text = util_calloc(1, textsz + 1); + + if (ZSTD_isError((textsz = ZSTD_decompress(text, textsz, zstd, zstdsz)))) { + free(zstd); + free(text); + return NULL; + } + + free(zstd); + + return text; +}