From 459cd2d7715baea5a56090b3331f0b9b95e832c2 Mon Sep 17 00:00:00 2001 From: "M.Shulhan" Date: Mon, 30 Mar 2009 16:45:54 +0700 Subject: vos: - add set PROCESS_TEMPORARY_DIRECTORY; for setting temporary directories used by sort process. - fix command-line parsing argument. --- op/vos_StmtSet.c | 3 ++- op/vos_String.c | 8 +++---- proc/vos_parser.c | 44 +++++++++++++++++++++++++++++++++------ proc/vos_set.c | 60 +++++++++++++++++++++++++++++++++++++++++++++++++++++ proc/vos_set.h | 2 ++ proc/vos_sort.c | 43 +++++++++++++++++++++++++++++++------- type/vos_TStmtSet.h | 1 + type/vos_TVos.h | 8 ++++++- vos.c | 33 ++++++++++++++++++++++++++--- vos.h | 1 + 10 files changed, 180 insertions(+), 23 deletions(-) diff --git a/op/vos_StmtSet.c b/op/vos_StmtSet.c index d4e91e6..0057e8d 100644 --- a/op/vos_StmtSet.c +++ b/op/vos_StmtSet.c @@ -5,7 +5,8 @@ const char *_stmt_set[N_SET] = { "PROCESS_COMPARE_CASE_SENSITIVE\0", "PROCESS_COMPARE_CASE_NOTSENSITIVE\0", "PROCESS_MAX\0", - "PROCESS_MAX_ROW\0" + "PROCESS_MAX_ROW\0", + "PROCESS_TEMPORARY_DIRECTORY\0" }; int stmtset_create(struct Stmt **stmt) diff --git a/op/vos_String.c b/op/vos_String.c index 679e243..ca13453 100644 --- a/op/vos_String.c +++ b/op/vos_String.c @@ -64,7 +64,6 @@ int str_append_c(struct String *str, const int c) */ int str_append(struct String *S, const char *str) { - int i; int len; if (! S) @@ -79,10 +78,11 @@ int str_append(struct String *S, const char *str) if (! S->buf) return E_MEM; } - for (i = 0; i < len; i++) - S->buf[S->idx++] = str[i]; - S->buf[S->idx] = '\0'; + memcpy(&S->buf[S->idx], str, len); + + S->idx += len; + S->buf[S->idx] = '\0'; return 0; } diff --git a/proc/vos_parser.c b/proc/vos_parser.c index ac57bf4..dfad97e 100644 --- a/proc/vos_parser.c +++ b/proc/vos_parser.c @@ -133,7 +133,9 @@ static int stmt_get_output_from(struct StmtMeta **smeta, struct Stmt *stmt, static int parsing_SET(struct Stmt **stmt, struct LL **ptok) { - int s = PSET_START; + int l = 0; + int s = PSET_START; + struct String *tmp = 0; while ((*ptok) && s != PSET_DONE) { switch (s) { @@ -159,9 +161,39 @@ static int parsing_SET(struct Stmt **stmt, struct LL **ptok) s = PSET_VALUE; break; case PSET_VALUE: - (*stmt)->set->value = (*ptok)->str; - (*ptok)->str = 0; - s = PSET_END; + if ((*stmt)->set->var_idx != SET_PROC_TMP_DIR) { + (*stmt)->set->value = (*ptok)->str; + (*ptok)->str = 0; + s = PSET_END; + break; + } + + /* (*stmt)->set->var_idx == SET_PROC_TMP_DIR */ + s = str_create(&tmp); + if (s) + return s; + + while ((*ptok) && strcmp((*ptok)->str, ";") != 0) { + if (strcmp((*ptok)->str, ":") == 0) { + str_append_c(tmp, ':'); + } else { + str_append_c(tmp, '"'); + str_append(tmp, (*ptok)->str); + + /* does it end with '/' ? */ + l = strlen((*ptok)->str) - 1; + if (l >= 0 && (*ptok)->str[l] != '/') + str_append_c(tmp, '/'); + + str_append_c(tmp, '"'); + } + (*ptok) = (*ptok)->next; + } + + (*stmt)->set->value = tmp->buf; + tmp->buf = 0; + s = PSET_DONE; + str_destroy(&tmp); break; case PSET_END: if ((*ptok)->str[0] != ';') { @@ -1139,7 +1171,7 @@ int vos_parsing(struct Stmt **stmt, const char *script) switch (FCURC(F)) { case CH_NEWLINE: line_no++; - case ' ': case '\b': case '\f': case '\r': case '\t': + case ' ': case '\b': case '\f': case '\r': case '\t': case '\v': break; @@ -1268,7 +1300,7 @@ int vos_parsing(struct Stmt **stmt, const char *script) F->idx++; } - if (_vos.debug & DBG_PARSER) { + if (_vos.debug & DBG_SCRIPT) { ll_print(ls_tok); } diff --git a/proc/vos_set.c b/proc/vos_set.c index 7fc3899..ec00f4d 100644 --- a/proc/vos_set.c +++ b/proc/vos_set.c @@ -1,5 +1,60 @@ #include "proc/vos_set.h" +static int set_proc_tmp_dir_value(char *tmp) +{ + int idx = 0; + int s = 0; + struct String *path = 0; + + s = str_create(&path); + if (s) + return s; + + if (*tmp == ':') { + tmp++; + idx = _vos.proc_tmp_dir->last->num; + } else + ll_destroy(&_vos.proc_tmp_dir); + + while (*tmp) { + /* skip '"' */ + tmp++; + + while(*tmp && *tmp != '"') { + s = str_append_c(path, *tmp); + if (s) + return s; + tmp++; + } + + /* does user have write access ? */ + s = access(path->buf, W_OK); + if (s == 0) { + idx++; + s = ll_add(&_vos.proc_tmp_dir, idx, path->buf); + if (s) + return s; + } + + /* skip '"' */ + tmp++; + + /* skip ':' */ + if (*tmp && *tmp == ':') + tmp++; + else + break; + + str_prune(path); + } + + _vos.p_proc_tmp_dir = _vos.proc_tmp_dir; + + str_destroy(&path); + + return 0; +} + int vos_process_set(struct StmtSet *set) { switch (set->var_idx) { @@ -28,6 +83,11 @@ int vos_process_set(struct StmtSet *set) if (! _vos.proc_max_row) _vos.proc_max_row = VOS_DEF_PROC_MAX_ROW; break; + + case SET_PROC_TMP_DIR: + set_proc_tmp_dir_value(set->value); + break; } + return 0; } diff --git a/proc/vos_set.h b/proc/vos_set.h index bf3432f..5d2445d 100644 --- a/proc/vos_set.h +++ b/proc/vos_set.h @@ -1,7 +1,9 @@ #ifndef _VOS_SET_H #define _VOS_SET_H 1 +#include #include "vos.h" +#include "op/vos_LL.h" #include "op/vos_StmtSet.h" int vos_process_set(struct StmtSet *set); diff --git a/proc/vos_sort.c b/proc/vos_sort.c index 70b5078..6e2029c 100644 --- a/proc/vos_sort.c +++ b/proc/vos_sort.c @@ -120,19 +120,44 @@ static struct Record * sort(struct Record *rows, unsigned long n_row) static int sort_write(struct ProcSort *psort, struct Record *rows) { - int s = 0; - char *tmp = 0; - struct File *Fout = 0; + int s = 0; + char *rndm_name = 0; + struct String *tmp = 0; + struct File *Fout = 0; + + str_create(&tmp); do { - s = str_raw_randomize(VOS_SORT_TMP_FORMAT, &tmp); + /* get a path to temporary directory */ + if (_vos.proc_max > 1) { + do { + s = pthread_mutex_trylock(&_vos.proc_tmp_dir_lock); + } while (s); + } + + str_append(tmp, _vos.p_proc_tmp_dir->str); + + _vos.p_proc_tmp_dir = _vos.p_proc_tmp_dir->next; + if (! _vos.p_proc_tmp_dir) + _vos.p_proc_tmp_dir = _vos.proc_tmp_dir; + + if (_vos.proc_max > 1) { + pthread_mutex_unlock(&_vos.proc_tmp_dir_lock); + } + + /* get random file name */ + s = str_raw_randomize(VOS_SORT_TMP_FORMAT, &rndm_name); if (s) - return s; + goto err; - s = file_open(&Fout, tmp, FOPEN_WO); + str_append(tmp, rndm_name); + + s = file_open(&Fout, tmp->buf, FOPEN_WO); if (s == 0) - ll_add(&psort->lsout, psort->tid, tmp); - free(tmp); + ll_add(&psort->lsout, psort->tid, tmp->buf); + + str_prune(tmp); + free(rndm_name); } while (s == E_FILE_EXIST); s = record_write(rows, Fout, psort->sort->out->fields); @@ -140,6 +165,8 @@ static int sort_write(struct ProcSort *psort, struct Record *rows) file_write(Fout); file_close(&Fout); +err: + str_destroy(&tmp); return s; } diff --git a/type/vos_TStmtSet.h b/type/vos_TStmtSet.h index 849a370..59275bf 100644 --- a/type/vos_TStmtSet.h +++ b/type/vos_TStmtSet.h @@ -7,6 +7,7 @@ enum _set_idx { SET_PROC_CMP_CASE_NOTSENSITIVE, SET_PROC_MAX, SET_PROC_MAX_ROW, + SET_PROC_TMP_DIR, N_SET }; diff --git a/type/vos_TVos.h b/type/vos_TVos.h index a22ff65..8e4a9a4 100644 --- a/type/vos_TVos.h +++ b/type/vos_TVos.h @@ -1,6 +1,9 @@ #ifndef _VOS_T_H #define _VOS_T_H +#include +#include "vos_TLL.h" + /** * @desc: * - DBG_SCRIPT : don't process the script @@ -31,12 +34,15 @@ struct Vos { int e_nparm0; /* first error parameter for number */ int e_nparm1; /* second error parameter for number */ int proc_cmp_case; /* use compare case sensitive ? */ + int proc_max; /* maximum process for sort */ unsigned long file_buf_size; /* size of buffer for file */ - unsigned long proc_max; /* maximum process for sort */ unsigned long proc_max_row; /* maximum row for each process */ + pthread_mutex_t proc_tmp_dir_lock; char *script; /* vos script */ char *e_sparm0; /* first error parameter (string) */ char *e_sparm1; /* second error parameter (string) */ + struct LL *proc_tmp_dir; /* process temporary directories */ + struct LL *p_proc_tmp_dir; }; #endif diff --git a/vos.c b/vos.c index 40f0d5a..c71cb27 100644 --- a/vos.c +++ b/vos.c @@ -48,15 +48,17 @@ static int vos_init(int argc, char **argv) return E_VOS_PARAM; s = strcmp(argv[i], "-d"); - if (s == 0) + if (s == 0) { s = ARG_DEBUG_VALUE; - else + i++; + } else s = ARG_VOS_SCRIPT; break; case ARG_DEBUG_VALUE: _vos.debug = strtol(argv[i], 0, 0); s = ARG_VOS_SCRIPT; + i++; break; case ARG_VOS_SCRIPT: @@ -64,11 +66,16 @@ static int vos_init(int argc, char **argv) s = ARG_DONE; break; } - i++; } if (s != ARG_DONE) return E_VOS_PARAM; + s = ll_add(&_vos.proc_tmp_dir, 1, VOS_DEF_PROC_TMP_DIR); + if (s) + return s; + + pthread_mutex_init(&_vos.proc_tmp_dir_lock, 0); + _vos.p_proc_tmp_dir = _vos.proc_tmp_dir; _vos.file_buf_size = VOS_DEF_FILE_BUF_SIZE; _vos.proc_cmp_case = VOS_DEF_PROC_CMP_CASE; _vos.proc_max = VOS_DEF_PROC_MAX; @@ -79,6 +86,19 @@ static int vos_init(int argc, char **argv) return 0; } +static void vos_print() +{ + printf(" vos debug : %d\n", _vos.debug); + printf(" vos file buffer size : %ld\n", _vos.file_buf_size); + printf(" vos process compare case : %s\n", _vos.proc_cmp_case ? + "not sensitive" : + "sensitive"); + printf(" vos process maximum : %d\n", _vos.proc_max); + printf(" vos process maximum row : %ld\n", _vos.proc_max_row); + printf(" vos process temporary dir:\n"); + ll_print(_vos.proc_tmp_dir); +} + static int vos_process(struct Stmt *stmt) { int s; @@ -87,6 +107,9 @@ static int vos_process(struct Stmt *stmt) switch (stmt->type) { case STMT_SET: vos_process_set(stmt->set); + + if (_vos.debug & DBG_PARSER) + vos_print(); break; case STMT_LOAD: s = vos_process_load(stmt); @@ -175,6 +198,10 @@ err: if (_vos.e_sparm1) free(_vos.e_sparm1); out: + if (_vos.proc_tmp_dir) + ll_destroy(&_vos.proc_tmp_dir); + + pthread_mutex_destroy(&_vos.proc_tmp_dir_lock); stmt_destroy(&stmt); return s; diff --git a/vos.h b/vos.h index cc1e3dc..f29a3ab 100644 --- a/vos.h +++ b/vos.h @@ -8,6 +8,7 @@ #define VOS_DEF_PROC_CMP_CASE CMP_CASE_SENSITIVE #define VOS_DEF_PROC_MAX 2 #define VOS_DEF_PROC_MAX_ROW 100000 +#define VOS_DEF_PROC_TMP_DIR "/tmp/" #define VOS_SORT_OUT_FORMAT "sort.XXXXXXXX" #define VOS_SORT_TMP_FORMAT "tmp.sort.XXXXXXXX" #define VOS_JOIN_OUT_FORMAT "join.XXXXXXXX" -- cgit v1.3