Commit 44a8715c authored by Hardy Simpson's avatar Hardy Simpson

feature: enable reload conf period & fsync period

parent 014a6869
......@@ -53,10 +53,11 @@
--- 1.0.1 ---
[o] 增加同步写入到硬盘选项,打开后极为费时
--- 1.0.2 ---
[o] 基于日志笔数自动刷新配置
[o] 基于日志笔数自动做sync到硬盘操作
--- 1.0.3 ---
[p] 使用valgrind测试性能
[ ] 基于日志笔数自动刷新配置
[ ] 基于日志笔数自动做sync到硬盘操作
[ ] 增加man age, df, 案例
[ ] 和rsyslog对接的问题
[ ] 测试多线程的效率
......
AC_PREREQ(2.59)
AC_INIT([zlog], [1.0.2], [HardySimpson1984@gmail.com])
AC_INIT([zlog], [1.0.3], [HardySimpson1984@gmail.com])
AM_INIT_AUTOMAKE([foreign -Wall -Werror])
AC_CONFIG_SRCDIR([libzlog/zlog.h])
AC_CONFIG_HEADER([config.h])
......
......@@ -1259,6 +1259,38 @@ my_dog.=DEBUG >syslog, LOG_LOCAL0; simple
my_mice.* @user_define;
\end_layout
\begin_layout Standard
有关单位:当设置内存大小或者大数字时,可以设置1k 5GB 4M这样的单位:
\end_layout
\begin_layout LyX-Code
# 1k => 1000 bytes
\end_layout
\begin_layout LyX-Code
# 1kb => 1024 bytes
\end_layout
\begin_layout LyX-Code
# 1m => 1000000 bytes
\end_layout
\begin_layout LyX-Code
# 1mb => 1024*1024 bytes
\end_layout
\begin_layout LyX-Code
# 1g => 1000000000 bytes
\end_layout
\begin_layout LyX-Code
# 1gb => 1024*1024*1024 byte
\end_layout
\begin_layout Standard
单位是大小写不敏感的,所以1GB 1Gb 1gB是等效的。
\end_layout
\begin_layout Section
全局参数
\end_layout
......@@ -1296,6 +1328,16 @@ strict init
是false的时候,zlog_init()会忽略错误的格式和规则。 这个参数默认为true。
\end_layout
\begin_layout Itemize
reload conf period
\end_layout
\begin_layout Standard
这个选项让zlog能在一段时间间隔后自动重载配置文件。重载的间隔以每进程写日志的次数来定义。当写日志次数到了一定值后,内部将会调用zlog_reload()进行
重载。每次zlog_reload()或者zlog_init()之后重新计数累加。因为zlog_reload()是原子性的,重载失败继续用当前的配置信息,所以自动
重载是安全的。默认值是0,自动重载是关闭的。
\end_layout
\begin_layout Itemize
buffer min
\end_layout
......@@ -1424,7 +1466,7 @@ file perms
\end_layout
\begin_layout Standard
这个指定了创建日志文件的缺省访问权限。默认为600,只允许当前用户读写。必须注意的是最后的产生的日志文件的权限为
这个指定了创建日志文件的缺省访问权限。必须注意的是最后的产生的日志文件的权限为
\begin_inset Quotes eld
\end_inset
......@@ -1432,7 +1474,60 @@ file perms
\begin_inset Quotes erd
\end_inset
& ~umask。
& ~umask。默认为600,只允许当前用户读写。
\end_layout
\begin_layout Itemize
fsync period
\end_layout
\begin_layout Standard
在每条规则写了一定次数的日志到文件后,zlog会调用fsync(3)来让操作系统马上把数据写到硬盘。次数是每条规则单独统计的,并且在zlog_reload()后
会被清0。必须指出的是,在日志文件名是动态生成或者被转档的情况下,zlog不能保证把所有文件都搞定,他只fsync()那个时候刚刚write()的文件描述符。这
提供了写日志速度和数据安全性之间的平衡。例子:
\end_layout
\begin_layout LyX-Code
$ time ./test_press_zlog 10 10000
\end_layout
\begin_layout LyX-Code
real 0m0.742s
\end_layout
\begin_layout LyX-Code
user 0m0.770s
\end_layout
\begin_layout LyX-Code
sys 0m0.640s
\end_layout
\begin_layout LyX-Code
$ time ./test_press_zlog 10 10000 #fsync period = 1K
\end_layout
\begin_layout LyX-Code
real 0m4.190s
\end_layout
\begin_layout LyX-Code
user 0m1.080s
\end_layout
\begin_layout LyX-Code
sys 0m1.240s
\end_layout
\begin_layout Standard
如果你极度在乎安全而不是速度的话,用同步IO文件,见
\begin_inset CommandInset ref
LatexCommand ref
reference "ite:同步IO文件"
\end_inset
。默认值是0,由操作系统来决定什么时候刷缓存到文件。
\end_layout
\begin_layout Section
......@@ -4650,6 +4745,13 @@ out.log(new create)
\end_deeper
\begin_layout Itemize
同步IO文件
\begin_inset CommandInset label
LatexCommand label
name "ite:同步IO文件"
\end_inset
\end_layout
\begin_deeper
......@@ -4662,7 +4764,7 @@ out.log(new create)
\begin_inset Quotes erd
\end_inset
就打开了同步IO选项。在打开文件(open)的时候,会以O_SYNC选项打开。这个选项极为耗时:
就打开了同步IO选项。在打开文件(open)的时候,会以O_SYNC选项打开,这时候每次写日志操作都会等操作系统把数据写到硬盘后才返回。这个选项极为耗时:
\end_layout
\begin_layout LyX-Code
......@@ -4678,7 +4780,7 @@ user 0m1.030s
\end_layout
\begin_layout LyX-Code
sys 0m1.080s
sys 0m1.080s
\end_layout
\begin_layout LyX-Code
......@@ -4694,7 +4796,7 @@ user 0m2.570s
\end_layout
\begin_layout LyX-Code
sys 0m6.950s
sys 0m6.950s
\end_layout
\end_deeper
......
......@@ -1234,6 +1234,10 @@ Most actions of zlog library are dependent upon configure file: where to
strict init = true
\end_layout
\begin_layout LyX-Code
reload conf period = 1M
\end_layout
\begin_layout LyX-Code
buffer min = 1024
\end_layout
......@@ -1254,6 +1258,10 @@ default format = "%d(%F %T.%l) %-6P (%c:%F:%L) - %m%n"
file perms = 600
\end_layout
\begin_layout LyX-Code
fsync period = 1K
\end_layout
\begin_layout LyX-Code
\end_layout
......@@ -1318,6 +1326,39 @@ my_dog.=DEBUG >syslog, LOG_LOCAL0; simple
my_mice.* @user_define;
\end_layout
\begin_layout Standard
Note on units: when memory size or large number is needed, it is possible
to specify it in the usual form of 1k 5GB 4M and so forth:
\end_layout
\begin_layout LyX-Code
# 1k => 1000 bytes
\end_layout
\begin_layout LyX-Code
# 1kb => 1024 bytes
\end_layout
\begin_layout LyX-Code
# 1m => 1000000 bytes
\end_layout
\begin_layout LyX-Code
# 1mb => 1024*1024 bytes
\end_layout
\begin_layout LyX-Code
# 1g => 1000000000 bytes
\end_layout
\begin_layout LyX-Code
# 1gb => 1024*1024*1024 byte
\end_layout
\begin_layout Standard
units are case insensitive so 1GB 1Gb 1gB are all the same.
\end_layout
\begin_layout Section
Global
\end_layout
......@@ -1360,6 +1401,21 @@ is false, zlog_init() will omit error syntax of formats and rules.
The default is true.
\end_layout
\begin_layout Itemize
reload conf period
\end_layout
\begin_layout Standard
This parameter supports zlog libraray reload configure file automaticlly
after a period, which is measured by number of log times per process.
When the number reaches the value, it calls zlog_reload() internally.
The number is reset to zero at the last zlog_reload() or zlog_init().
As zlog_reload() is atomic, when zlog_reload() fails, zlog still run with
the current configure.
So automaticlly reloading configure is safe.
The defualt is 0, which means never reload automaticlly.
\end_layout
\begin_layout Itemize
buffer min
\end_layout
......@@ -1529,7 +1585,6 @@ file perms
\begin_layout Standard
This specifies all log file permission when they are created.
The default is 600, which just allowed user read and write.
Note that it is affect by user's umask.
The final file permission will be
\begin_inset Quotes eld
......@@ -1540,6 +1595,70 @@ file perms
\end_inset
& ~umask.
The default is 600, which just allowed user read and write.
\end_layout
\begin_layout Itemize
fsync period
\end_layout
\begin_layout Standard
After a number of log times per rule(to file only), zlog will call fsync(3)
after write() to tells the Operating System to actually write data on disk
immediately.
The number is counted by each rule and will be reset to 0 after zlog_reload().
Note that when the file's path is dynamic generated or is rotated, zlog
do not guarantee fsync() hit all files.
It just fsync() the file descriptor that has been write() to at the boundary
time.
It offers a balance between speed and data safety.
A example:
\end_layout
\begin_layout LyX-Code
$ time ./test_press_zlog 10 10000
\end_layout
\begin_layout LyX-Code
real 0m0.742s
\end_layout
\begin_layout LyX-Code
user 0m0.770s
\end_layout
\begin_layout LyX-Code
sys 0m0.640s
\end_layout
\begin_layout LyX-Code
$ time ./test_press_zlog 10 10000 #fsync period = 1K
\end_layout
\begin_layout LyX-Code
real 0m4.190s
\end_layout
\begin_layout LyX-Code
user 0m1.080s
\end_layout
\begin_layout LyX-Code
sys 0m1.240s
\end_layout
\begin_layout Standard
If you want extreme safety but not care about speed, using synchronous I/O
file, see
\begin_inset CommandInset ref
LatexCommand ref
reference "ite:synchronous-I/O-file"
\end_inset
.The defualt is 0, which means let the operating system flush the output
buffer when it wants.
\end_layout
\begin_layout Section
......@@ -4244,7 +4363,7 @@ aa.!debug
User can define his own level, see
\begin_inset CommandInset ref
LatexCommand ref
reference "sec:Define-new-level"
reference "sec:User-defined-Level"
\end_inset
......@@ -4908,12 +5027,20 @@ So the oldest log has the biggest serial number.
\end_deeper
\begin_layout Itemize
synchronous I/O file
\begin_inset CommandInset label
LatexCommand label
name "ite:synchronous-I/O-file"
\end_inset
\end_layout
\begin_deeper
\begin_layout Standard
Putting a minus signe '-' set the synchronous I/O opition.
log file is opend with O_SYNC.
log file is opend with O_SYNC and every single log action will return until
Operation System write data to disk.
It is painfully slow:
\end_layout
......@@ -4930,7 +5057,7 @@ user 0m1.030s
\end_layout
\begin_layout LyX-Code
sys 0m1.080s
sys 0m1.080s
\end_layout
\begin_layout LyX-Code
......@@ -4946,7 +5073,7 @@ user 0m2.570s
\end_layout
\begin_layout LyX-Code
sys 0m6.950s
sys 0m6.950s
\end_layout
\end_deeper
......@@ -5149,7 +5276,8 @@ confpath
\bar default
is NULL, it reloads the last configure file zlog_init() or zlog_reload()
specified.
If zlog_reload() failed, the last configuration in memory will remain unchanged.
If zlog_reload() failed, the current configuration in memory will remain
unchanged.
So zlog_reload() is atomic.
\end_layout
......
[global]
strict init = true
reload conf period = 10M
buffer min = 1024
buffer max = 2MB
rotate lock file = /tmp/zlog.lock
#rotate lock file = /tmp/zlog.lock
rotate lock file = self
default format = "%d(%F %T.%l) %-6V (%c:%F:%L) - %m%n"
file perms = 600
reload conf = 10M
sync file = 1K
fsync period = 1K
[levels]
TRACE = 10
......
......@@ -39,32 +39,11 @@
#define ZLOG_CONF_DEFAULT_BUF_SIZE_MIN 1024
#define ZLOG_CONF_DEFAULT_BUF_SIZE_MAX (2 * 1024 * 1024)
#define ZLOG_CONF_DEFAULT_FILE_PERMS 0600
#define ZLOG_CONF_DEFAULT_RELOAD_CONF_PERIOD 0
#define ZLOG_CONF_DEFAULT_FSYNC_PERIOD 0
#define ZLOG_CONF_BACKUP_ROTATE_LOCK_FILE "/tmp/zlog.lock"
/*******************************************************************************/
struct zlog_conf_s {
char file[MAXLEN_PATH + 1];
char mtime[20 + 1];
int strict_init;
size_t buf_size_min;
size_t buf_size_max;
char rotate_lock_file[MAXLEN_CFG_LINE + 1];
zlog_rotater_t *rotater;
zc_arraylist_t *levels;
char default_format_line[MAXLEN_CFG_LINE + 1];
zlog_format_t *default_format;
unsigned int file_perms;
zc_arraylist_t *formats;
zc_arraylist_t *rules;
};
void zlog_conf_profile(zlog_conf_t * a_conf, int flag)
{
int i;
......@@ -83,6 +62,8 @@ void zlog_conf_profile(zlog_conf_t * a_conf, int flag)
zlog_format_profile(a_conf->default_format, flag);
}
zc_profile(flag, "---file perms[0%o]---", a_conf->file_perms);
zc_profile(flag, "---reload conf period[%ld]---", a_conf->reload_conf_period);
zc_profile(flag, "---fsync period[%ld]---", a_conf->fsync_period);
zc_profile(flag, "---rotate lock file[%s]---", a_conf->rotate_lock_file);
if (a_conf->rotater) zlog_rotater_profile(a_conf->rotater, flag);
......@@ -162,6 +143,8 @@ zlog_conf_t *zlog_conf_new(char *conf_file)
}
strcpy(a_conf->default_format_line, ZLOG_CONF_DEFAULT_FORMAT);
a_conf->file_perms = ZLOG_CONF_DEFAULT_FILE_PERMS;
a_conf->reload_conf_period = ZLOG_CONF_DEFAULT_RELOAD_CONF_PERIOD;
a_conf->fsync_period = ZLOG_CONF_DEFAULT_FSYNC_PERIOD;
/* set default configuration end */
a_conf->levels = zlog_level_list_new();
......@@ -169,8 +152,7 @@ zlog_conf_t *zlog_conf_new(char *conf_file)
zc_error("zlog_level_list_new fail");
rc = -1;
goto zlog_conf_new_exit;
}
}
a_conf->formats = zc_arraylist_new((zc_arraylist_del_fn) zlog_format_del);
if (!a_conf->formats) {
zc_error("zc_arraylist_new fail");
......@@ -232,7 +214,8 @@ static int zlog_conf_build_without_file(zlog_conf_t * a_conf)
a_conf->levels,
a_conf->default_format,
a_conf->formats,
a_conf->file_perms);
a_conf->file_perms,
a_conf->fsync_period);
if (!default_rule) {
zc_error("zlog_rule_new fail");
return -1;
......@@ -451,6 +434,11 @@ static int zlog_conf_parse_line(zlog_conf_t * a_conf, char *line, int *section)
} else if (STRCMP(word_1, ==, "default") && STRCMP(word_2, ==, "format")) {
/* so the input now is [format = "xxyy"], fit format's style */
strcpy(a_conf->default_format_line, line + nread);
} else if (STRCMP(word_1, ==, "reload") &&
STRCMP(word_2, ==, "conf") && STRCMP(word_3, ==, "period")) {
a_conf->reload_conf_period = zc_parse_byte_size(value);
} else if (STRCMP(word_1, ==, "fsync") && STRCMP(word_2, ==, "period")) {
a_conf->fsync_period = zc_parse_byte_size(value);
} else {
zc_error("name[%s] is not any one of global options", name);
if (a_conf->strict_init) return -1;
......@@ -483,7 +471,9 @@ static int zlog_conf_parse_line(zlog_conf_t * a_conf, char *line, int *section)
a_conf->levels,
a_conf->default_format,
a_conf->formats,
a_conf->file_perms);
a_conf->file_perms,
a_conf->fsync_period);
if (!a_rule) {
zc_error("zlog_rule_new fail [%s]", line);
if (a_conf->strict_init) return -1;
......@@ -505,25 +495,3 @@ static int zlog_conf_parse_line(zlog_conf_t * a_conf, char *line, int *section)
return 0;
}
/*******************************************************************************/
zc_arraylist_t *zlog_conf_get_rules(zlog_conf_t *a_conf)
{
zc_assert(a_conf, NULL);
return a_conf->rules;
}
void zlog_conf_get_buf_size(zlog_conf_t *a_conf,
size_t * buf_size_min, size_t * buf_size_max)
{
zc_assert(a_conf,);
zc_assert(buf_size_min,);
zc_assert(buf_size_max,);
*buf_size_min = a_conf->buf_size_min;
*buf_size_max = a_conf->buf_size_max;
}
char *zlog_conf_get_file(zlog_conf_t *a_conf)
{
zc_assert(a_conf, NULL);
return a_conf->file;
}
......@@ -22,16 +22,33 @@
#include "zc_defs.h"
#include "format.h"
#include "rotater.h"
typedef struct zlog_conf_s zlog_conf_t;
typedef struct zlog_conf_s {
char file[MAXLEN_PATH + 1];
char mtime[20 + 1];
int strict_init;
size_t buf_size_min;
size_t buf_size_max;
char rotate_lock_file[MAXLEN_CFG_LINE + 1];
zlog_rotater_t *rotater;
char default_format_line[MAXLEN_CFG_LINE + 1];
zlog_format_t *default_format;
unsigned int file_perms;
size_t fsync_period;
size_t reload_conf_period;
zc_arraylist_t *levels;
zc_arraylist_t *formats;
zc_arraylist_t *rules;
} zlog_conf_t;
zlog_conf_t *zlog_conf_new(char *conf_file);
void zlog_conf_del(zlog_conf_t * a_conf);
void zlog_conf_profile(zlog_conf_t * a_conf, int flag);
zc_arraylist_t *zlog_conf_get_rules(zlog_conf_t *a_conf);
void zlog_conf_get_buf_size(zlog_conf_t *a_conf,
size_t * buf_size_min, size_t * buf_size_max);
char *zlog_conf_get_file(zlog_conf_t *a_conf);
#endif
......@@ -450,13 +450,11 @@ int zlog_rotater_rotate(zlog_rotater_t *a_rotater,
rc = stat(file_path, &info);
if (rc) {
zc_error("stat [%s] fail, errno[%d]", file_path, errno);
return -1;
zc_warn("stat [%s] fail, errno[%d], maybe in rotating", file_path, errno);
return 0;
} else {
if (info.st_size + msg_len < file_max_size) {
/* file not so big, return */
return 0;
}
/* file not so big, return */
if (info.st_size + msg_len < file_max_size) return 0;
}
rd = zlog_rotater_trylock(a_rotater);
......
......@@ -25,6 +25,8 @@
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <signal.h>
#include <unistd.h>
#include "rule.h"
#include "format.h"
......@@ -59,6 +61,9 @@ struct zlog_rule_s {
long file_max_size;
int file_max_count;
size_t fsync_period;
volatile sig_atomic_t fsync_count;
zc_arraylist_t *levels;
int syslog_facility;
......@@ -78,7 +83,7 @@ void zlog_rule_profile(zlog_rule_t * a_rule, int flag)
zlog_spec_t *a_spec;
zc_assert(a_rule,);
zc_profile(flag, "---rule:[%p][%s%c%d]-[%s|%p(O%o:%d:%ld*%d)][%d][%s:%s:%p];[%p]---",
zc_profile(flag, "---rule:[%p][%s%c%d]-[%s|%p(%d:%ld*%d)][%d][%s:%s:%p];[%p]---",
a_rule,
a_rule->category,
a_rule->compare_char,
......@@ -87,9 +92,7 @@ void zlog_rule_profile(zlog_rule_t * a_rule, int flag)
a_rule->file_path,
a_rule->dynamic_file_specs,
a_rule->file_perms,
a_rule->file_open_flags,
a_rule->file_max_size,
a_rule->file_max_count,
......@@ -109,6 +112,7 @@ void zlog_rule_profile(zlog_rule_t * a_rule, int flag)
}
/*******************************************************************************/
static int zlog_rule_output_static_file_single(zlog_rule_t * a_rule, zlog_thread_t * a_thread)
{
int rc = 0;
......@@ -140,6 +144,11 @@ static int zlog_rule_output_static_file_single(zlog_rule_t * a_rule, zlog_thread
return -1;
}
if (a_rule->fsync_period && ++a_rule->fsync_count > a_rule->fsync_period) {
a_rule->fsync_count = 0;
if (fsync(fd)) zc_error("fsync[%d] fail, errno[%d]", fd, errno);
}
rc = close(fd);
if (rc < 0) {
zc_error("write fail, maybe cause by write, errno[%d]", errno);
......@@ -180,6 +189,11 @@ static int zlog_rule_output_static_file_rotate(zlog_rule_t * a_rule, zlog_thread
return -1;
}
if (a_rule->fsync_period && ++a_rule->fsync_count > a_rule->fsync_period) {
a_rule->fsync_count = 0;
if (fsync(fd)) zc_error("fsync[%d] fail, errno[%d]", fd, errno);
}
rc = close(fd);
if (rc < 0) {
zc_error("write fail, maybe cause by write, errno[%d]", errno);
......@@ -259,6 +273,11 @@ static int zlog_rule_output_dynamic_file_single(zlog_rule_t * a_rule, zlog_threa
return -1;
}
if (a_rule->fsync_period && ++a_rule->fsync_count > a_rule->fsync_period) {
a_rule->fsync_count = 0;
if (fsync(fd)) zc_error("fsync[%d] fail, errno[%d]", fd, errno);
}
rc = close(fd);
if (rc < 0) {