shayla

a simple yet fast static site builder
Log | Files | Refs | Submodules | README | LICENSE | git clone https://git.ne02ptzero.me/git/shayla

commit 15b023254970d957753e8de7c05687847fad4a92
parent 96b3e3e28f4dfa6300c05a52a1cb02d96ac5e611
Author: Louis Solofrizzo <lsolofrizzo@online.net>
Date:   Mon, 27 Aug 2018 20:50:33 +0200

Files: Add recursive file listing

This commit introduces numerous things. The main one is to be able to
list every file in a directory with 'list_dir', using linked lists for
that. The implementation is to be safe, and seems so at the moment
(Valgrind does not shout at me). File links are properly handled,
exception made for kernel file systems (/proc, /dev, etc) which are
hardcoded to be ignored at the moment.

I've also added creation / destruction method for the main context
(shayla_t) and files (file_t), it should be now simple and clear.

Last, there is the (almost) entire linux linked list implementation,
adapted for user space and my needs.

Signed-off-by: Louis Solofrizzo <lsolofrizzo@online.net>

Diffstat:
MCMakeLists.txt | 2++
Adir.c | 220+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Adir.h | 32++++++++++++++++++++++++++++++++
Afile.h | 120+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Alist.c | 148+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Alist.h | 128+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Mlog.c | 14++++++++++++++
Mlog.h | 7+++++++
Mmain.c | 21++++++---------------
Ashayla.h | 58++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Astr.h | 32++++++++++++++++++++++++++++++++
11 files changed, 767 insertions(+), 15 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt @@ -43,6 +43,8 @@ add_executable(${SDOW_NAME} main.c #parser.c #parse_header.c log.c + list.c + dir.c # site.c #./sundown/src/autolink.c #./sundown/src/buffer.c diff --git a/dir.c b/dir.c @@ -0,0 +1,220 @@ +/** + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <https://www.gnu.org/licenses/>. +*/ + +#include <dir.h> +#include <file.h> +#include <dirent.h> +#include <unistd.h> +#include <sys/stat.h> +#include <errno.h> +#include <str.h> + +static const char *reserved_directories[] = { + "/boot", + "/dev", + "/proc", + "/run", +}; + +static bool add_dir(list_head_t *head, const char *path) +{ + char real_path[PATH_MAX] = EMPTY_STR(); + file_t *ptr; + file_t *tmp; + + path = realpath(path, real_path); + if (path == NULL) + return true; + + STATIC_ARRAY_FOREACH(const char **dir, reserved_directories) + { + if (strcmp(*dir, path) == 0) + return true; + } + + ptr = file_new(path); + if (ptr == NULL) + { + PROGRESS_ERROR("Could not allocate memory for directory"); + return false; + } + + list_for_each_entry(tmp, head, list) + { + if (strcmp(tmp->path, path) == 0) + { + file_free(ptr); + return true; + } + } + + list_add_tail(&ptr->list, head); + return true; +} + +static bool add_file(list_head_t *files, const char *path, const char *ext, size_t *file_count) +{ + file_t *tmp = file_new(path); + + if (tmp == NULL) + return false; + + if (ext != NULL) + { + if (!str_ends_with(path, ext)) + return true; + } + + list_add_tail(&tmp->list, files); + (*file_count)++; + PROGRESS_UPDATE("Listing %zu files...", *file_count); + + return true; +} + +static bool follow_link(char *path, list_head_t *files, list_head_t *dir, const char *ext, size_t *file_count) +{ + char tg_path[PATH_MAX] = EMPTY_STR(); + char tg_full_path[PATH_MAX * 2] = EMPTY_STR(); + int fd; + struct stat buf = { 0 }; + + if (readlink(path, tg_path, sizeof(tg_path)) == -1) + { + PROGRESS_ERROR("'%s': readlink() error", path); + return false; + } + + if (tg_path[0] != '/') + { + size_t i; + for (i = strlen(path) - 1; i > 0 && path[i] != '/'; i--) + ; + + if (i > 0) + path[i] = '\0'; + snprintf(tg_full_path, sizeof(tg_full_path), "%s/%s", path, tg_path); + if (i > 0) + path[i] = '/'; + } + else + strcpy(tg_full_path, tg_path); + + fd = open(tg_full_path, O_RDONLY); + if (fd == -1) + { + PROGRESS_WARNING("Could not follow symlink '%s' to '%s'", path, tg_full_path); + return true; + } + + if (fstat(fd, &buf) == -1) + { + PROGRESS_ERROR("fstat() error\n"); + close(fd); + return false; + } + + if (buf.st_mode & S_IFDIR) + add_dir(dir, tg_full_path); + else if (buf.st_mode & S_IFLNK && + !(buf.st_mode & S_IFIFO || buf.st_mode & S_IFSOCK || + buf.st_mode & S_IFBLK || buf.st_mode & S_IFCHR)) + { + close(fd); + return follow_link(tg_full_path, files, dir, ext, file_count); + } + else if (buf.st_mode & S_IFREG) + { + if (!add_file(files, tg_full_path, ext, file_count)) + { + close(fd); + return false; + } + } + + close(fd); + return true; +} + +static bool read_dir(list_head_t *files, list_head_t *dir, const char *root, DIR *fd, const char *ext, size_t *file_count) +{ + struct dirent *ptr; + char path[PATH_MAX] = EMPTY_STR(); + + while ((ptr = readdir(fd))) + { + if (ptr->d_name[0] == '.') + continue ; + + snprintf(path, sizeof(path), "%s/%s", root, ptr->d_name); + if (ptr->d_type == DT_DIR) + { + if (!add_dir(dir, path)) + return false; + } + else if (ptr->d_type == DT_LNK) + { + if (!follow_link(path, files, dir, ext, file_count)) + return false; + } + else if (ptr->d_type == DT_REG) + { + if (!add_file(files, path, ext, file_count)) + return false; + } + } + + return true; +} + +bool list_dir(list_head_t *head, const char *dir_name, const char *ext) +{ + list_head_t dir; + file_t *tmp; + bool ret = false; + size_t file_count = 0; + + INIT_LIST_HEAD(&dir); + if (!add_dir(&dir, dir_name)) + return false; + + file_count = 0; + PROGRESS_BEGIN("Listing %zu files...", file_count, dir_name); + + list_for_each_entry(tmp, &dir, list) + { + DIR *fd = opendir(tmp->path); + + if (fd == NULL) + { + PROGRESS_WARNING("Cannot open %s: %s", tmp->path, strerror(errno)); + continue ; + } + + if (!read_dir(head, &dir, tmp->path, fd, ext, &file_count)) + { + closedir(fd); + goto end; + } + + closedir(fd); + } + + PROGRESS_DONE(); + ret = true; +end: + file_list_free(&dir); + return ret; +} diff --git a/dir.h b/dir.h @@ -0,0 +1,32 @@ +/** + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <https://www.gnu.org/licenses/>. +*/ + +#ifndef DIR_H +# define DIR_H + +# include <list.h> + +/*! + * \brief List every file in a directory + * + * \param[out] head Output linked list + * \param[in] dir_name Directory name + * \param[in] ext File extension to look for. Can be NULL + * + * \return true on success, false on failure + */ +bool list_dir(list_head_t *head, const char *dir_name, const char *ext); + +#endif /* DIR_H */ diff --git a/file.h b/file.h @@ -0,0 +1,120 @@ +/** + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <https://www.gnu.org/licenses/>. +*/ + +#ifndef FILE_H +# define FILE_H + +# include <list.h> +# include <log.h> +# include <time.h> +# include <stdlib.h> +# include <string.h> +# include <limits.h> + +typedef struct { + char path[PATH_MAX]; /*!< Path of the file */ + char *route; /*!< Web-Route of the post */ + char *title; /*!< Title of the post */ + char *summary; /*!< Summary of the post */ + bool is_listed; /*!< Wether or not the post is listed on the index */ + + list_head_t list; /*!< List member */ + time_t ts; /*!< Last update time of the file */ +} file_t; + +/*! + * \brief Destroy a file_t pointer + * + * \param[in,out] ptr Pointer to destroy + */ +static inline void file_dtr(file_t *ptr) +{ + UNUSED_ARG(ptr); +} + +/*! + * \brief Init a file_t pointer + * + * \param[in,out] ptr Pointer to init + * \param[in] path Path of the file + * + * \return true on success, false on failure + */ +static inline bool file_init(file_t *ptr, const char *path) +{ + if (strlen(path) > sizeof(ptr->path)) + { + DEBUG("The size of the path is greater than " STR(sizeof(ptr->path)) \ + ", cannot continue.\n"); + return false; + } + + strcpy(ptr->path, path); + return true; +} + +/*! + * \brief Free a file_t pointer + */ +static inline void file_free(file_t *ptr) +{ + if (ptr != NULL) + { + file_dtr(ptr); + free(ptr); + } +} + +/*! + * \brief Allocate and return a new file_t pointer + * + * \param[in] path Path of the file + * + * \return A freshly allocated pointer on success, NULL on failure + */ +static inline file_t *file_new(const char *path) +{ + file_t *result; + + result = calloc(1, sizeof(*result)); + if (result == NULL) + { + ERROR("File allocation failed\n"); + return NULL; + } + + if (!file_init(result, path)) + { + file_free(result); + return NULL; + } + + return result; +} + +/*! + * \brief Free a list of file_t + */ +static inline void file_list_free(list_head_t *files) +{ + file_t *tmp, *iter; + + list_for_each_entry_safe(tmp, iter, files, list) + { + file_free(tmp); + } +} + +#endif /* FILE_H */ diff --git a/list.c b/list.c @@ -0,0 +1,148 @@ +/** + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <https://www.gnu.org/licenses/>. +*/ + +#include <list.h> +#include <stdint.h> +#include <string.h> + +#define MAX_LIST_LENGTH_BITS 20 + +/** + * Returns a list organized in an intermediate format suited + * to chaining of merge() calls: null-terminated, no reserved or + * sentinel head node, "prev" links not maintained. + */ +static list_head_t *merge(void *priv, + int (*cmp)(void *priv, list_head_t *a, list_head_t *b), + list_head_t *a, list_head_t *b) +{ + list_head_t head, *tail = &head; + + while (a && b) + { + /* if equal, take 'a' -- important for sort stability */ + if ((*cmp)(priv, a, b) <= 0) + { + tail->next = a; + a = a->next; + } + else + { + tail->next = b; + b = b->next; + } + tail = tail->next; + } + tail->next = a?:b; + + return head.next; +} + +/** + * Combine final list merge with restoration of standard doubly-linked + * list structure. This approach duplicates code from merge(), but + * runs faster than the tidier alternatives of either a separate final + * prev-link restoration pass, or maintaining the prev links + * throughout. + */ +static void merge_and_restore_back_links(void *priv, + int (*cmp)(void *priv, list_head_t *a, list_head_t *b), + list_head_t *head, list_head_t *a, list_head_t *b) +{ + list_head_t *tail = head; + uint8_t count = 0; + + while (a && b) + { + /* if equal, take 'a' -- important for sort stability */ + if ((*cmp)(priv, a, b) <= 0) + { + tail->next = a; + a->prev = tail; + a = a->next; + } + else + { + tail->next = b; + b->prev = tail; + b = b->next; + } + tail = tail->next; + } + tail->next = a ? : b; + + do + { + /* + * In worst cases this loop may run many iterations. + * Continue callbacks to the client even though no + * element comparison is needed, so the client's cmp() + * routine can invoke cond_resched() periodically. + */ + if (unlikely(!(++count))) + (*cmp)(priv, tail->next, tail->next); + + tail->next->prev = tail; + tail = tail->next; + + } while (tail->next); + + tail->next = head; + head->prev = tail; +} + +void list_sort(void *priv, list_head_t *head, int (*cmp)(void *priv, list_head_t *a, list_head_t *b)) +{ + list_head_t *part[MAX_LIST_LENGTH_BITS + 1] = { 0 }; /* sorted partial lists last slot is a sentinel */ + int lev; /* index into part[] */ + int max_lev = 0; + list_head_t *list; + + if (list_empty(head)) + return; + + head->prev->next = NULL; + list = head->next; + + while (list) + { + list_head_t *cur = list; + list = list->next; + cur->next = NULL; + + for (lev = 0; part[lev]; lev++) + { + cur = merge(priv, cmp, part[lev], cur); + part[lev] = NULL; + } + + if (lev > max_lev) + { + if (unlikely(lev >= (int)(COUNT_OF(part) - 1))) + lev--; + + max_lev = lev; + } + part[lev] = cur; + } + + for (lev = 0; lev < max_lev; lev++) + { + if (part[lev]) + list = merge(priv, cmp, part[lev], list); + } + + merge_and_restore_back_links(priv, cmp, head, part[max_lev], list); +} diff --git a/list.h b/list.h @@ -0,0 +1,128 @@ +/** + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <https://www.gnu.org/licenses/>. +*/ + +#ifndef LIST_H +# define LIST_H + +# include <utils.h> + +typedef struct list_head { + struct list_head *next, *prev; +} list_head_t; + +# define list_for_each(pos, head) \ + for (pos = (head)->next; pos != (head); pos = pos->next) + +# define list_for_each_safe(pos, n, head) \ + for (pos = (head)->next, n = pos->next; pos != (head); \ + pos = n, n = pos->next) + +# define list_for_each_entry(pos, head, member) \ + for (pos = list_entry((head)->next, typeof(*pos), member); \ + &pos->member != (head); \ + pos = list_entry(pos->member.next, typeof(*pos), member)) + +# define list_for_each_entry_safe(pos, n, head, member) \ + for (pos = list_first_entry(head, typeof(*pos), member), \ + n = list_next_entry(pos, member); \ + &pos->member != (head); \ + pos = n, n = list_next_entry(n, member)) + +# define list_entry(ptr, type, member) container_of(ptr, type, member) +# define list_first_entry(ptr, type, member) list_entry((ptr)->next, type, member) +# define list_last_entry(ptr, type, member) list_entry((ptr)->prev, type, member) +# define list_next_entry(pos, member) list_entry((pos)->member.next, typeof(*(pos)), member) +# define list_prev_entry(pos, member) list_entry((pos)->member.prev, typeof(*(pos)), member) + +static inline void INIT_LIST_HEAD(list_head_t *list) +{ + list->next = list; + list->prev = list; +} + +static inline void list_add(list_head_t *new, + list_head_t *prev, + list_head_t *next) +{ + next->prev = new; + new->next = next; + new->prev = prev; + prev->next = new; +} + +static inline void list_add_tail(list_head_t *new, list_head_t *head) +{ + list_add(new, head->prev, head); +} + +static inline void __list_del(list_head_t *prev, list_head_t *next) +{ + next->prev = prev; +} + +static inline void list_del(list_head_t *entry) +{ + __list_del(entry->prev, entry->next); +} + +/*! + * \brief Check wheter or not a list is empty + * + * \param[in] head Head of the list + * + * \return true if the list is empty, false otherwise + */ +static inline bool list_empty(const struct list_head *head) +{ + return head->next == head; +} + +/*! + * \brief Count the number of elements in a list + * + * \param[in] head Head of the list + * + * \return Size of the list + */ +static inline size_t list_count(const list_head_t *head) +{ + list_head_t *iter; + size_t i = 0; + + list_for_each(iter, head) + i++; + + return i; +} + +/** + * \brief Sort a list + * + * \param[in] priv private data, opaque to list_sort(), passed to cmp + * \param[in,out] head the list to sort + * \param[in] cmp the elements comparison function + * + * This function implements "merge sort", which has O(nlog(n)) + * complexity. + * + * The comparison function @cmp must return a negative value if @a + * should sort before @b, and a positive value if @a should sort after + * @b. If @a and @b are equivalent, and their original relative + * ordering is to be preserved, @cmp must return 0. + */ +void list_sort(void *priv, list_head_t *head, int (*cmp)(void *priv, list_head_t *a, list_head_t *b)); + + +#endif /*LIST_H */ diff --git a/log.c b/log.c @@ -88,6 +88,20 @@ void progress_update(const char *str, ...) va_end(ap); } +void progress_warning(const char *str, ...) +{ + va_list ap; + + va_start(ap, str); + + pthread_mutex_lock(&msg_lock); + fprintf(stdout, "\r" SYMBOL_WARNING " "); + vfprintf(stdout, str, ap); + pthread_mutex_unlock(&msg_lock); + + va_end(ap); +} + void progress_end(bool result) { run = false; diff --git a/log.h b/log.h @@ -83,6 +83,12 @@ void progress_begin(const char *str, ...); void progress_update(const char *str, ...); /*! + * \brief Send a warning during a progress thread + */ +#define PROGRESS_WARNING(str, ...) progress_warning(str "\n", ##__VA_ARGS__) +void progress_warning(const char *str, ...); + +/*! * \brief Terminate a spinner progress thread * * \param[in] result Status of the termination (true = all, false = error) @@ -102,4 +108,5 @@ void progress_update(const char *str, ...); } while (0) void progress_end(bool result); + #endif /* LOG_H */ diff --git a/main.c b/main.c @@ -14,26 +14,17 @@ */ #include <log.h> +#include <shayla.h> +#include <dir.h> int main(void) { - DEBUG("Alu !\n"); - INFO("info\n"); - WARNING("Warning\n"); - ERROR("Error!\n"); - SET_DEBUG(true); - DEBUG("Oui!\n"); + shayla_t ctx = { 0 }; - PROGRESS_BEGIN("Yes."); + shayla_init(&ctx); - for (size_t i = 0; i < 100000; i++) - { - PROGRESS_UPDATE("Count: %zu", i); - usleep(200); - } + list_dir(&ctx.posts, ".", ".md"); - PROGRESS_DONE(); - - FATAL("ava ?\n"); + shayla_dtr(&ctx); return 0; } diff --git a/shayla.h b/shayla.h @@ -0,0 +1,58 @@ +/** + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <https://www.gnu.org/licenses/>. +*/ + +#ifndef SHAYLA_H +# define SHAYLA_H + +# include <list.h> +# include <file.h> + +typedef struct { + list_head_t posts; /*!< List of posts (file_t) */ + list_head_t styles; /*!< List of styles (file_t) */ + list_head_t layouts; /*!< List of layouts (file_t) */ + list_head_t img; /*!< List of images (file_t ) */ + + const char *posts_dir; /*!< Directory of the posts */ + const char *styles_dir; /*!< Style directory */ + const char *layouts_dir; /*!< Layouts directory */ + const char *img_dir; /*!< Image directory */ + + const char *title; /*!< Title of the site */ + const char *dest; /*!< Destination directory */ + const char *root; /*!< Root path of the website */ + const char *url; /*!< URL of the website */ + const char *favicon; /*!< Favicon of the website */ +} shayla_t; + +/*! + * \brief Init a shayla_t pointer + * + * \param[in] ptr Pointer to init + */ +static inline void shayla_init(shayla_t *ptr) +{ + INIT_LIST_HEAD(&ptr->posts); + INIT_LIST_HEAD(&ptr->styles); + INIT_LIST_HEAD(&ptr->layouts); + INIT_LIST_HEAD(&ptr->img); +} + +static inline void shayla_dtr(shayla_t *ptr) +{ + file_list_free(&ptr->posts); +} + +#endif /* SHAYLA_H */ diff --git a/str.h b/str.h @@ -0,0 +1,32 @@ +/** + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <https://www.gnu.org/licenses/>. +*/ + +#ifndef STR_H +# define STR_H + +/*! + * \brief Check if a string ends with another string + * + * \param[in] str Hastack + * \param[in] ext Needle + * + * \return true if end match, false otherwise + */ +static inline bool str_ends_with(const char *str, const char *ext) +{ + return strcmp(str + strlen(str) - strlen(ext), ext) == 0; +} + +#endif /* STR_H */