您介意测试以下verifier.c吗？
/* SPDX-License-Identifier: CC0-1.0 */
#define _POSIX_C_SOURCE 200809L
#include <stdlib.h>
#include <unistd.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <sys/time.h>
#include <sys/resource.h>
#include <pthread.h>
#include <limits.h>
#include <fcntl.h>
#include <ftw.h>
#include <string.h>
#include <errno.h>
#include <stdio.h>
/* Singly-linked list of paths. */
struct path_list {
struct path_list *next;
int err;
char path[];
};
/* File content verifier. Returns 0 if OK, errno error code otherwise. */
static int verify_data(const unsigned char *data, const size_t size)
{
/*
* TODO!
*/
(void)data; (void)size; /* Remove; these just silence warnings about unused parameters. */
return 0;
}
static inline void free_path_list(struct path_list *list)
{
while (list) {
struct path_list *curr = list;
list = list->next;
curr->next = NULL;
curr->err = -1;
free(curr);
}
}
/* Number of descriptors excluded in the calculations */
#ifndef RESERVE_DESCRIPTORS
#define RESERVE_DESCRIPTORS 10
#endif
/* Minimum number of descriptors for nftw() */
#ifndef MINIMUM_NFTW_DESCRIPTORS
#define MINIMUM_NFTW_DESCRIPTORS 16
#endif
/* Maximum number of descriptors for nftw()
- recommended number is twice the maximum depth of the tree. */
#ifndef MAXIMUM_NFTW_DESCRIPTORS
#define MAXIMUM_NFTW_DESCRIPTORS 256
#endif
static int verify_tree_add(const char *, const struct stat *, int);
static void *verify_tree_worker(void *);
static pthread_t *verify_worker = NULL;
static size_t verify_workers = 0;
static unsigned long verify_count_ok = 0;
static unsigned long verify_count = 0;
static pthread_mutex_t verify_lock = PTHREAD_MUTEX_INITIALIZER;
static pthread_cond_t verify_more = PTHREAD_COND_INITIALIZER;
static struct path_list *verify_list = NULL; /* Paths to be verified */
static struct path_list *verify_fail = NULL; /* Paths that failed the check */
static int verify_done = 0; /* Set when path buffer completed */
/* Verify all regular files in the specified tree using 'workers' threads.
Returns the list of files that failed verification.
Always sets errno: to zero if success, to nonzero if error.
*/
static struct path_list *verify_tree(const char *root, const size_t workers)
{
struct path_list *retval;
pthread_attr_t attrs;
struct rlimit limit;
long maxfd;
size_t i;
int result;
/* Sanity check. */
if (!root || *root == '\0' || workers < 1) {
errno = EINVAL;
return NULL;
}
/* Maximum number of file descriptors */
if (getrlimit(RLIMIT_NOFILE, &limit) == -1)
return NULL;
/* If limited, try to up the soft limit to the hard limit. */
if (limit.rlim_max != RLIM_INFINITY && limit.rlim_cur < limit.rlim_max) {
limit.rlim_cur = limit.rlim_max;
setrlimit(RLIMIT_NOFILE, &limit);
}
/* Find out the maximum number of file descriptors. */
maxfd = sysconf(_SC_OPEN_MAX);
if (limit.rlim_cur != RLIM_INFINITY && (long)limit.rlim_cur < maxfd)
maxfd = (long)limit.rlim_cur;
/* Subtract the number of reserved descriptors and those needed by workers */
if (maxfd > (long)(workers + RESERVE_DESCRIPTORS))
maxfd -= workers + RESERVE_DESCRIPTORS;
/* and limit to the configured range. */
if (maxfd < MINIMUM_NFTW_DESCRIPTORS)
maxfd = MINIMUM_NFTW_DESCRIPTORS;
if (maxfd > MAXIMUM_NFTW_DESCRIPTORS)
maxfd = MAXIMUM_NFTW_DESCRIPTORS;
/* Initialize worker information. */
verify_workers = 0;
verify_worker = malloc(workers * sizeof verify_worker[0]);
if (!verify_worker) {
errno = ENOMEM;
return NULL;
}
pthread_mutex_init(&verify_lock, NULL);
pthread_cond_init(&verify_more, NULL);
verify_list = NULL;
verify_fail = NULL;
verify_done = 0;
/* Clear the counters. */
verify_count = 0;
verify_count_ok = 0;
/* Threads need very little stack. */
pthread_attr_init(&attrs);
pthread_attr_setstacksize(&attrs, 2*PTHREAD_STACK_MIN);
/* Start the worker threads. */
for (i = 0; i < workers; i++) {
result = pthread_create(verify_worker + verify_workers, &attrs, verify_tree_worker, NULL);
if (!result)
verify_workers++;
}
/* Discard the thread creation attribute set. */
pthread_attr_destroy(&attrs);
/* No workers started? */
if (verify_workers < 1) {
free(verify_worker);
verify_workers = 0;
verify_worker = NULL;
verify_done = -1;
pthread_cond_destroy(&verify_more);
pthread_mutex_destroy(&verify_lock);
/* Insufficient resources */
errno = EAGAIN;
return NULL;
}
/* Start directory scan. */
result = ftw(root, verify_tree_add, maxfd);
if (result) {
/* Failed; abort. Save errno. */
if (result == -1)
result = errno;
/* Signal threads to cancel work. */
pthread_mutex_lock(&verify_lock);
verify_done = -1;
pthread_cond_broadcast(&verify_more);
pthread_mutex_unlock(&verify_lock);
/* Reap worker threads. */
for (i = 0; i < verify_workers; i++)
pthread_join(verify_worker[i], NULL);
/* Grab the failed paths list. */
retval = verify_fail;
verify_fail = NULL;
/* Discard any paths not verified. */
free_path_list(verify_list);
verify_list = NULL;
/* Cleanup. */
free(verify_worker);
verify_workers = 0;
verify_worker = NULL;
pthread_cond_destroy(&verify_more);
pthread_mutex_destroy(&verify_lock);
errno = result;
return retval;
}
/* Success. Signal threads that no more paths will be added. */
pthread_mutex_lock(&verify_lock);
verify_done = 1;
pthread_cond_broadcast(&verify_more);
pthread_mutex_unlock(&verify_lock);
/* Reap worker threads. */
for (i = 0; i < verify_workers; i++)
pthread_join(verify_worker[i], NULL);
/* Will return the failed paths list. */
retval = verify_fail;
verify_fail = NULL;
/* Prepend all unprocessed paths -- should be none! -- to failed list. */
while (verify_list) {
struct path_list *curr = verify_list;
verify_list = curr->next;
curr->next = retval;
retval = curr;
}
/* Cleanup. */
free(verify_worker);
verify_workers = 0;
verify_worker = NULL;
pthread_cond_destroy(&verify_more);
pthread_mutex_destroy(&verify_lock);
/* Success. */
errno = 0;
return retval;
}
/* Add regular files to the work pool. */
static int verify_tree_add(const char *path, const struct stat *info, int typeflag)
{
struct path_list *item;
size_t path_len;
/* Ignore all but regular files. */
if (typeflag != FTW_F)
return 0;
/* Ignore zero-sized files. */
if (!info->st_size)
return 0;
/* Ignore NULL and empty paths, although they shouldn't happen. */
if (!path || !*path)
return 0;
path_len = strlen(path);
item = malloc(sizeof (struct path_list) + path_len + 1);
if (!item)
return ENOMEM;
/* Copy the path, including the end-of-string '\0'. */
memcpy(item->path, path, path_len + 1);
/* This path item should be OK. */
item->err = 0;
/* Prepend to the path list. */
pthread_mutex_lock(&verify_lock);
item->next = verify_list;
verify_list = item;
/* Signal a waiting worker. */
pthread_cond_signal(&verify_more);
/* Add to count. */
verify_count++;
pthread_mutex_unlock(&verify_lock);
return 0;
}
/* Work pool worker thread. */
static void *verify_tree_worker(void *unused __attribute__((unused)))
{
unsigned char data[128];
size_t have;
ssize_t bytes;
int fd, result;
struct path_list *curr;
pthread_mutex_lock(&verify_lock);
while (verify_done >= 0) {
/* No paths in the list? */
if (!verify_list) {
/* All done? */
if (verify_done)
break;
/* No, wait for a new path. */
pthread_cond_wait(&verify_more, &verify_lock);
continue;
}
/* Extract path from the list. */
curr = verify_list;
verify_list = curr->next;
/* Release the mutex for the duration of the verification work. */
pthread_mutex_unlock(&verify_lock);
/* Open the file to be verified. */
fd = open(curr->path, O_RDONLY | O_CLOEXEC);
if (fd == -1) {
curr->err = errno;
/* Append to failed list. */
pthread_mutex_lock(&verify_lock);
curr->next = verify_fail;
verify_fail = curr;
continue;
}
/* Read the initial part of the file, or entire file if it fits in the buffer. */
have = 0;
while (have < sizeof data) {
bytes = read(fd, data + have, sizeof data - have);
if (bytes > 0) {
have += bytes;
} else
if (bytes == 0) {
/* All read */
break;
} else
if (bytes != -1) {
/* Bug. Mark as an I/O error. */
curr->err = EIO;
have = 0;
break;
} else
if (errno != EINTR && errno != EAGAIN && errno != EWOULDBLOCK) {
curr->err = errno;
have = 0;
break;
}
}
if (close(fd) == -1) {
/* This does not normally occur, but let's be thorough. */
have = 0;
if (!curr->err)
curr->err = errno;
}
/* Empty file or error? */
if (!have) {
/* Append to failed list. */
pthread_mutex_lock(&verify_lock);
curr->next = verify_fail;
verify_fail = curr;
continue;
}
/* Verify file contents. */
result = verify_data(data, have);
if (result) {
/* Append to failed list. */
pthread_mutex_lock(&verify_lock);
curr->err = result;
curr->next = verify_fail;
verify_fail = curr;
continue;
}
/* Verification successful. */
free(curr);
pthread_mutex_lock(&verify_lock);
verify_count_ok++;
}
pthread_mutex_unlock(&verify_lock);
return NULL;
}
static int parse_size(const char *src, size_t *to)
{
const char *end;
unsigned long val;
if (!src)
return errno = EINVAL;
errno = 0;
end = src;
val = strtoul(src, (char **)(&end), 0);
if (errno)
return errno;
if (end == src)
return errno = EINVAL;
while (*end == '\t' || *end == '\n' || *end == '\v' ||
*end == '\f' || *end == '\r' || *end == ' ')
end++;
if (*end)
return errno = EINVAL;
if ((unsigned long)(size_t)(val) != val)
return errno = ERANGE;
if (to)
*to = (size_t)val;
return 0;
}
int main(int argc, char *argv[])
{
struct path_list *list, *curr;
size_t threads;
if (argc != 3 || !strcmp(argv[1], "-h") || !strcmp(argv[2], "--help")) {
const char *self = (argc > 0 && argv[0]) ? argv[0] : "(this)";
fprintf(stderr, "\n");
fprintf(stderr, "Usage: %s DIRECTORY THREADS\n", self);
fprintf(stderr, "\n");
fprintf(stderr, "This program scans all regular files in DIRECTORY\n");
fprintf(stderr, "and all its subdirectories, using THREADS threads\n");
fprintf(stderr, "in parallel.\n");
fprintf(stderr, "\n");
return EXIT_FAILURE;
}
if (parse_size(argv[2], &threads) || threads < 1) {
fprintf(stderr, "%s: Invalid number of threads.\n", argv[2]);
return EXIT_FAILURE;
}
list = verify_tree(argv[1], threads);
if (errno) {
fprintf(stderr, "Verification failed: %s.\n", strerror(errno));
for (curr = list; curr; curr = curr->next)
printf(" %s: (%s)\n", curr->path, strerror(curr->err));
free_path_list(list);
return EXIT_FAILURE;
}
if (list) {
fprintf(stderr, "Verification complete: %lu of %lu files ok.\n", verify_count_ok, verify_count);
for (curr = list; curr; curr = curr->next)
printf(" %s: (%s)\n", curr->path, strerror(curr->err));
free_path_list(list);
return EXIT_FAILURE;
}
printf("Verified all %lu files successfully!\n", verify_count_ok);
return EXIT_SUCCESS;
}
要编译，我推荐使用
gcc -Wall -Wextra -O2 verifier.c -lpthread -o verifier
要运行，请使用类似
./verifier /tree/with/files 64
其中第一个参数是要验证的目录树，第二个参数是要使用的工作线程数。
上面的程序实现了一个简单的基于线程池的单进程多线程验证器。 （您应该在verify_data() 函数中实现实际的数据验证；现在，它假定如果文件可以读取，那就没问题。注意verify_tree_add() 函数忽略空文件；您可能希望删除该检查。以这种方式测试程序对我来说更容易。）
主线程使用ftw() 进行目录树扫描，回调函数提取完整路径，并添加到要打开和检查的路径列表中。
每个工作线程从列表中获取下一个路径，打开文件，读取前 128 个字节（或整个文件，如果文件更小），并验证内容。如果有错误，则将路径添加到失败的路径列表中（返回给调用者）；否则释放。
这种方法背后的想法是，大多数工作线程将在 read() 调用中阻塞，从而允许内核优化 I/O 请求顺序并最大限度地增加正在运行的页面数。
这根本没有优化，您可能需要添加额外的保护（不要让路径列表无限增长），但底层模式应该允许您最大化吞吐量。
（在将新邮件保存到 Maildir 邮箱时，例如邮件传输代理可以使用类似的模式，其中每封邮件都是一个单独的文件。为了确保每个文件都已访问存储设备，fsync()（或至少@987654332 @) 需要在描述符上调用，但这可能需要相当长的时间。让一个工作线程在fsync()/fdatasync() 调用上执行保存和阻塞允许邮件传输代理同时处理其他消息。它在吞吐量上有很大的不同。）
为了测试，我比较了
sudo sh -c 'sync ; echo 3 > /proc/sys/vm/drop_caches ; sync'
time find /usr -type f -ls > /dev/null
实时耗时约 30 秒；和
sudo sh -c 'sync ; echo 3 > /proc/sys/vm/drop_caches ; sync'
time ./verifier /usr 60
这需要大约 50 秒的实时时间。
在这两个命令中，sudo 部分清除页表缓存（详见man 5 proc 部分/proc/sys/vm/drop_caches），因此当计算机尚未缓存任何文件内容并且相对静止时，这两种情况对应wrt。 I/O（sync 部分）。
运行 cache-hot，这两个命令分别需要大约 8 和 30 秒。
（为了测试，我使用了一个验证器来检查第一个字节是否为 0x1c，最后读取的字节是否为 0xfe，以确保编译器没有优化任何内容。换句话说，这些测试确实读取了最初的 128 个字节，或者整个文件（如果较小）。）
这是 HP EliteBook 820 G1 笔记本电脑，配备 Intel i5-4200U 处理器和三星 SSD，/usr 下有 989610（非空）文件。显然，SSD 的性能远胜于旋转磁盘存储，尤其是对于这种工作负载（小文件，即大量小的 I/O 传输块），但对于 100-2 亿个文件来说，三个小时（或更长时间）对我来说听起来有点过分了。