Commit 3f2fb81e authored by Hardy Simpson's avatar Hardy Simpson

optmize: cache static file's stream

parent b27c33d2
......@@ -56,12 +56,13 @@
[o] 基于日志笔数自动刷新配置
[o] 基于日志笔数自动做sync到硬盘操作
--- 1.0.3 ---
[o] 缓存静态文件的fd,优化
[o] 测试多线程的效率
[o] 缓存静态文件的FILE*和fd,优化
--- 1.0.4 ---
[p] 使用valgrind测试性能
[ ] 增加man age, df, 案例
[ ] 和rsyslog对接的问题
[ ] 测试多线程的效率
[ ] hzlog的可定制话
[ ] 分类匹配的可定制化
[ ] 兼容性问题
......
AC_PREREQ(2.59)
AC_INIT([zlog], [1.0.3], [HardySimpson1984@gmail.com])
AC_INIT([zlog], [1.0.4], [HardySimpson1984@gmail.com])
AM_INIT_AUTOMAKE([foreign -Wall -Werror])
AC_CONFIG_SRCDIR([libzlog/zlog.h])
AC_CONFIG_HEADER([config.h])
......
......@@ -163,13 +163,6 @@ zlog_buf_t *zlog_buf_new(size_t buf_size_min, size_t buf_size_max,
}
}
/*******************************************************************************/
void zlog_buf_restart(zlog_buf_t * a_buf)
{
a_buf->end = a_buf->start;
return;
}
/*******************************************************************************/
int zlog_buf_printf(zlog_buf_t * a_buf, const char *format, ...)
{
......
......@@ -40,7 +40,10 @@ zlog_buf_t *zlog_buf_new(size_t buf_size_min, size_t buf_size_max,
void zlog_buf_del(zlog_buf_t * a_buf);
void zlog_buf_profile(zlog_buf_t * a_buf, int flag);
void zlog_buf_restart(zlog_buf_t * a_buf);
#define zlog_buf_restart(a_buf) do { \
a_buf->end = a_buf->start; \
} while(0);
int zlog_buf_printf(zlog_buf_t * a_buf, const char *format, ...);
int zlog_buf_vprintf(zlog_buf_t * a_buf, const char *format, va_list args);
int zlog_buf_append(zlog_buf_t * a_buf, const char *str, size_t str_len);
......
......@@ -55,6 +55,7 @@ struct zlog_rule_s {
char file_path[MAXLEN_PATH + 1];
zc_arraylist_t *dynamic_file_specs;
int static_file_descriptor;
FILE *static_file_stream;
unsigned int file_perms;
int file_open_flags;
......@@ -83,7 +84,7 @@ void zlog_rule_profile(zlog_rule_t * a_rule, int flag)
zlog_spec_t *a_spec;
zc_assert(a_rule,);
zc_profile(flag, "---rule:[%p][%s%c%d]-[%s,%p,%d(%d:%ld*%d)][%d][%s:%s:%p];[%p]---",
zc_profile(flag, "---rule:[%p][%s%c%d]-[%s,%p,(%d,%p)(%d:%ld*%d)][%d][%s:%s:%p];[%p]---",
a_rule,
a_rule->category,
a_rule->compare_char,
......@@ -92,6 +93,7 @@ void zlog_rule_profile(zlog_rule_t * a_rule, int flag)
a_rule->file_path,
a_rule->dynamic_file_specs,
a_rule->static_file_descriptor,
a_rule->static_file_stream,
a_rule->file_open_flags,
a_rule->file_max_size,
......@@ -117,7 +119,7 @@ void zlog_rule_profile(zlog_rule_t * a_rule, int flag)
static int zlog_rule_output_static_file_single(zlog_rule_t * a_rule, zlog_thread_t * a_thread)
{
int rc;
ssize_t nwrite;
size_t nwrite;
rc = zlog_format_gen_msg(a_rule->format, a_thread);
if (rc) {
......@@ -125,17 +127,18 @@ static int zlog_rule_output_static_file_single(zlog_rule_t * a_rule, zlog_thread
return -1;
}
nwrite = write(a_rule->static_file_descriptor,
a_thread->msg_buf->start, a_thread->msg_buf->end - a_thread->msg_buf->start);
if (nwrite < 0) {
zc_error("write fail, errno[%d]", errno);
close(a_rule->static_file_descriptor);
nwrite = fwrite(a_thread->msg_buf->start, a_thread->msg_buf->end - a_thread->msg_buf->start,
1, a_rule->static_file_stream);
if (nwrite != 1) {
zc_error("fwrite fail, errno[%d]", errno);
return -1;
}
if (a_rule->fsync_period && ++a_rule->fsync_count > a_rule->fsync_period) {
a_rule->fsync_count = 0;
if (fsync(a_rule->static_file_descriptor)) {
if (fflush(a_rule->static_file_stream)) {
zc_error("fflush[%p] fail, errno[%d]", a_rule->static_file_stream, errno);
} else if (fsync(a_rule->static_file_descriptor)) {
zc_error("fsync[%d] fail, errno[%d]", a_rule->static_file_descriptor, errno);
}
}
......@@ -146,7 +149,7 @@ static int zlog_rule_output_static_file_single(zlog_rule_t * a_rule, zlog_thread
static int zlog_rule_output_static_file_rotate(zlog_rule_t * a_rule, zlog_thread_t * a_thread)
{
int rc;
ssize_t nwrite;
size_t nwrite;
size_t len;
rc = zlog_format_gen_msg(a_rule->format, a_thread);
......@@ -156,16 +159,17 @@ static int zlog_rule_output_static_file_rotate(zlog_rule_t * a_rule, zlog_thread
}
len = a_thread->msg_buf->end - a_thread->msg_buf->start;
nwrite = write(a_rule->static_file_descriptor, a_thread->msg_buf->start, len);
if (nwrite < 0) {
zc_error("write fail, errno[%d]", errno);
close(a_rule->static_file_descriptor);
nwrite = fwrite(a_thread->msg_buf->start, len, 1, a_rule->static_file_stream);
if (nwrite != 1) {
zc_error("fwrite fail, errno[%d]", errno);
return -1;
}
if (a_rule->fsync_period && ++a_rule->fsync_count > a_rule->fsync_period) {
a_rule->fsync_count = 0;
if (fsync(a_rule->static_file_descriptor)) {
if (fflush(a_rule->static_file_stream)) {
zc_error("fflush[%p] fail, errno[%d]", a_rule->static_file_stream, errno);
} else if (fsync(a_rule->static_file_descriptor)) {
zc_error("fsync[%d] fail, errno[%d]", a_rule->static_file_descriptor, errno);
}
}
......@@ -180,9 +184,9 @@ static int zlog_rule_output_static_file_rotate(zlog_rule_t * a_rule, zlog_thread
return -1;
} else if (rc > 0) {
/* rotate succ, reopen fd to avoid write to old file*/
rc = close(a_rule->static_file_descriptor);
if (rc < 0) {
zc_error("close fail, maybe cause by write, errno[%d]", errno);
rc = fclose(a_rule->static_file_stream);
if (rc) {
zc_error("fclose fail, maybe cause by write, errno[%d]", errno);
}
a_rule->static_file_descriptor = open(a_rule->file_path,
......@@ -192,6 +196,12 @@ static int zlog_rule_output_static_file_rotate(zlog_rule_t * a_rule, zlog_thread
return -1;
}
a_rule->static_file_stream = fdopen(a_rule->static_file_descriptor, "a");
if (!a_rule->static_file_stream) {
zc_error("fdopen fd[%d] fail, errno[%d]", a_rule->static_file_descriptor, errno);
return -1;
}
}
return 0;
......@@ -669,6 +679,13 @@ zlog_rule_t *zlog_rule_new(char *line,
goto zlog_rule_new_exit;
}
a_rule->static_file_stream = fdopen(a_rule->static_file_descriptor, "a");
if (!a_rule->static_file_stream) {
rc = -1;
zc_error("fdopen fd[%d] fail, errno[%d]", a_rule->static_file_descriptor, errno);
goto zlog_rule_new_exit;
}
} else {
zlog_spec_t *a_spec;
......@@ -753,9 +770,9 @@ void zlog_rule_del(zlog_rule_t * a_rule)
zc_arraylist_del(a_rule->dynamic_file_specs);
a_rule->dynamic_file_specs = NULL;
}
if (a_rule->static_file_descriptor) {
if (close(a_rule->static_file_descriptor) < 0) {
zc_error("close fail, maybe cause by write, errno[%d]", errno);
if (a_rule->static_file_stream) {
if (fclose(a_rule->static_file_stream)) {
zc_error("fclose fail, maybe cause by write, errno[%d]", errno);
}
}
free(a_rule);
......
......@@ -12,7 +12,9 @@ exe = \
test_mdc \
test_record \
test_press_zlog \
test_press_zlog2 \
test_press_printf \
test_press_printf2 \
test_press_syslog \
test_syslog \
test_default \
......
......@@ -23,18 +23,25 @@
#include <sys/wait.h>
#include <unistd.h>
#include <pthread.h>
#include <sys/stat.h>
#include <fcntl.h>
#include "zlog.h"
FILE *fp;
int fd;
static long loop_count;
void * work(void *ptr)
{
long j = loop_count;
static char aa[] = "2012-05-16 17:24:58.282603 INFO 22471:test_press_zlog.c:33 loglog\n";
while(j-- > 0) {
fprintf(fp, "2012-05-16 17:24:58.282603 INFO 22471:test_press_zlog.c:33 loglog\n");
// fprintf(fp, "2012-05-16 17:24:58.282603 INFO 22471:test_press_zlog.c:33 loglog\n");
fwrite(aa, sizeof(aa)-1, 1, fp);
// write(fd, aa, sizeof(aa)-1);
}
return 0;
}
......@@ -78,15 +85,13 @@ int main(int argc, char** argv)
exit(1);
}
fp = fopen("press.log", "a");
if (!fp) {
printf("fopen fail\n");
return 1;
}
fd = open("press.log", O_CREAT | O_WRONLY | O_APPEND, 0644);
fp = fdopen(fd, "a");
loop_count = atol(argv[3]);
test(atol(argv[1]), atol(argv[2]));
fclose(fp);
return 0;
}
......@@ -38,12 +38,19 @@ void * work(void *ptr)
char file[20];
sprintf(file, "press.%ld.log", (long)ptr);
/*
int fd;
fd = open(file, O_CREAT | O_WRONLY | O_APPEND , 0644);
*/
FILE *fp;
fp = fopen(file, "a");
while(j-- > 0) {
int fd;
fd = open(file, O_CREAT | O_WRONLY | O_APPEND , 0755);
write(fd, log, sizeof(log)-1);
close(fd);
//write(fd, log, sizeof(log)-1);
fwrite(log, sizeof(log)-1, 1, fp);
}
fclose(fp);
//close(fd);
return 0;
}
......
......@@ -5,14 +5,4 @@ file perms = 0777
#reload conf period = 1K
[rules]
my_cat.* "press.log"
cat0.* "press.0.log"
cat1.* "press.1.log"
cat2.* "press.2.log"
cat3.* "press.3.log"
cat4.* "press.4.log"
cat5.* "press.5.log"
cat6.* "press.6.log"
cat7.* "press.7.log"
cat8.* "press.8.log"
cat9.* "press.9.log"
*.* "press.log"
......@@ -61,7 +61,7 @@ int test(long process_count, long thread_count)
pthread_t tid[thread_count];
for (j = 0; j < thread_count; j++) {
pthread_create(&(tid[j]), NULL, work, (void*)j);
pthread_create(&(tid[j]), NULL, work, (void*)i);
}
for (j = 0; j < thread_count; j++) {
pthread_join(tid[j], NULL);
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment