diff --git a/plugins/in_tail/tail.c b/plugins/in_tail/tail.c index 5818f38f0ce..1679c968a2e 100644 --- a/plugins/in_tail/tail.c +++ b/plugins/in_tail/tail.c @@ -380,15 +380,6 @@ static int in_tail_init(struct flb_input_instance *in, /* Scan path */ flb_tail_scan(ctx->path_list, ctx); -#ifdef FLB_HAVE_SQLDB - /* Delete stale files that are not monitored from the database */ - ret = flb_tail_db_stale_file_delete(in, config, ctx); - if (ret == -1) { - flb_tail_config_destroy(ctx); - return -1; - } -#endif - if (ctx->read_newly_discovered_files_from_head) { /* * After the first scan (on start time), all new files discovered needs to be @@ -490,11 +481,66 @@ static int in_tail_init(struct flb_input_instance *in, } /* Pre-run callback / before the event loop */ +#ifdef FLB_HAVE_SQLDB +struct tail_db_cleanup_context { + struct flb_tail_config *ctx; + struct flb_config *config; +}; + +static int tail_db_inode_is_monitored(uint64_t inode, void *data) +{ + struct tail_db_cleanup_context *cleanup_context = data; + struct mk_list *input_head; + struct mk_list *file_head; + struct flb_input_instance *input; + struct flb_tail_config *tail_ctx; + struct flb_tail_file *file; + + mk_list_foreach(input_head, &cleanup_context->config->inputs) { + input = mk_list_entry(input_head, struct flb_input_instance, _head); + if (input->p != cleanup_context->ctx->ins->p || + input->context == NULL) { + continue; + } + + tail_ctx = input->context; + if (tail_ctx->db == NULL || + tail_ctx->db->handler != cleanup_context->ctx->db->handler) { + continue; + } + + mk_list_foreach(file_head, &tail_ctx->files_static) { + file = mk_list_entry(file_head, struct flb_tail_file, _head); + if (file->inode == inode) { + return FLB_TRUE; + } + } + } + + return FLB_FALSE; +} +#endif + static int in_tail_pre_run(struct flb_input_instance *ins, struct flb_config *config, void *in_context) { +#ifdef FLB_HAVE_SQLDB + int ret; + struct tail_db_cleanup_context cleanup_context; +#endif struct flb_tail_config *ctx = in_context; (void) ins; + (void) config; + +#ifdef FLB_HAVE_SQLDB + cleanup_context.ctx = ctx; + cleanup_context.config = config; + ret = flb_tail_db_cleanup(ctx, tail_db_inode_is_monitored, + &cleanup_context); + if (ret != 0) { + flb_plg_error(ctx->ins, "db: stale file cleanup failed"); + } +#endif return tail_signal_manager(ctx); } diff --git a/plugins/in_tail/tail_db.c b/plugins/in_tail/tail_db.c index 3bb5f0d0cab..cb98c3818ad 100644 --- a/plugins/in_tail/tail_db.c +++ b/plugins/in_tail/tail_db.c @@ -25,6 +25,7 @@ #include "tail_sql.h" #include "tail_file.h" +#include #include #include @@ -34,6 +35,11 @@ struct query_status { int64_t offset; }; +struct stale_file { + uint64_t id; + struct mk_list _head; +}; + static int db_apply_migration_if_needed(struct flb_tail_config *ctx, struct flb_sqldb *db, const char *sql) @@ -184,6 +190,106 @@ static int flb_tail_db_file_delete_by_id(struct flb_tail_config *ctx, return 0; } +static int stale_file_matches(struct flb_tail_config *ctx, + flb_tail_db_inode_check_fn inode_is_monitored, + void *data, + const char *path, uint64_t inode) +{ + int ret; + struct stat st; + +#ifdef FLB_SYSTEM_WINDOWS + if (ctx->windows_path_encoding == FLB_TAIL_WINDOWS_PATH_ENCODING_UTF8) { + ret = win32_stat_utf8(path, &st); + } + else { + ret = stat(path, &st); + } +#else + ret = stat(path, &st); +#endif + + if (ret == 0 && inode == (uint64_t) st.st_ino) { + return FLB_TRUE; + } + + if (inode_is_monitored != NULL && + inode_is_monitored(inode, data) == FLB_TRUE) { + return FLB_TRUE; + } + + if (ret == 0) { + return FLB_FALSE; + } + + if (errno == ENOENT || errno == ENOTDIR) { + return FLB_FALSE; + } + + return FLB_TRUE; +} + +static int stale_file_delete_missing(struct flb_tail_config *ctx, + flb_tail_db_inode_check_fn inode_is_monitored, + void *data, + sqlite3_stmt *stmt, int *deleted_count) +{ + int ret; + int result = 0; + const char *name; + struct mk_list stale_files; + struct mk_list *head; + struct mk_list *tmp; + struct stale_file *stale_file; + + mk_list_init(&stale_files); + *deleted_count = 0; + + while ((ret = sqlite3_step(stmt)) == SQLITE_ROW) { + name = (const char *) sqlite3_column_text(stmt, 1); + if (name == NULL || + stale_file_matches(ctx, inode_is_monitored, data, name, + sqlite3_column_int64(stmt, 2)) == FLB_TRUE) { + continue; + } + + stale_file = flb_malloc(sizeof(struct stale_file)); + if (stale_file == NULL) { + flb_errno(); + result = -1; + goto cleanup; + } + + stale_file->id = sqlite3_column_int64(stmt, 0); + mk_list_add(&stale_file->_head, &stale_files); + } + + if (ret != SQLITE_DONE) { + flb_plg_error(ctx->ins, "db: cannot query stale files: ret=%d", ret); + result = -1; + goto cleanup; + } + + mk_list_foreach(head, &stale_files) { + stale_file = mk_list_entry(head, struct stale_file, _head); + ret = flb_tail_db_file_delete_by_id(ctx, stale_file->id); + if (ret != 0) { + result = -1; + goto cleanup; + } + (*deleted_count)++; + } + +cleanup: + mk_list_foreach_safe(head, tmp, &stale_files) { + stale_file = mk_list_entry(head, struct stale_file, _head); + mk_list_del(&stale_file->_head); + flb_free(stale_file); + } + + return result; +} + /* * Check if an file inode exists in the database. * If the 'compare_filename' option is enabled, @@ -297,182 +403,6 @@ static int db_file_insert(struct flb_tail_file *file, struct flb_tail_config *ct return flb_sqldb_last_id(ctx->db); } -static int stmt_add_param_concat(struct flb_tail_config *ctx, - flb_sds_t *stmt_sql, uint64_t count) -{ - uint64_t idx; - flb_sds_t sds_tmp; - - sds_tmp = flb_sds_cat(*stmt_sql, SQL_STMT_START_PARAM, - SQL_STMT_START_PARAM_LEN); - if (sds_tmp == NULL) { - flb_plg_debug(ctx->ins, "error concatenating stmt_sql: param start"); - return -1; - } - *stmt_sql = sds_tmp; - - for (idx = 1; idx < count; idx++) { - sds_tmp = flb_sds_cat(*stmt_sql, SQL_STMT_ADD_PARAM, - SQL_STMT_ADD_PARAM_LEN); - if (sds_tmp == NULL) { - flb_plg_debug(ctx->ins, "error concatenating stmt_sql: add param"); - return -1; - } - - *stmt_sql = sds_tmp; - } - - sds_tmp = flb_sds_cat(*stmt_sql, SQL_STMT_PARAM_END, - SQL_STMT_PARAM_END_LEN); - if (sds_tmp == NULL) { - flb_plg_debug(ctx->ins, "error concatenating stmt_sql: param end"); - return -1; - } - *stmt_sql = sds_tmp; - - return 0; -} - -/* - * Scalable stale inode cleanup: use a temp table to avoid SQLite variable limits. - * - * The legacy implementation builds: - * DELETE ... WHERE inode NOT IN (?,?,?,...); - * which requires one bound parameter per inode and fails when the number of - * monitored files exceeds SQLITE_LIMIT_VARIABLE_NUMBER (commonly 32766 in our - * bundled SQLite, but can vary). - */ -static int flb_tail_db_stale_file_delete_temp_table(struct flb_tail_config *ctx, - uint64_t file_count, - int db_locked) -{ - int ret; - int changes; - int txn_started = FLB_FALSE; - sqlite3_stmt *stmt_insert_inode = NULL; - struct mk_list *head; - struct mk_list *tmp; - struct flb_tail_file *file; - - /* If there are no monitored files, delete everything from the DB table. */ - if (file_count == 0) { - ret = flb_sqldb_query(ctx->db, "DELETE FROM in_tail_files;", NULL, NULL); - if (ret != FLB_OK) { - flb_plg_error(ctx->ins, "db: cannot delete all stale inodes (no monitored files)"); - goto error; - } - - changes = sqlite3_changes(ctx->db->handler); - flb_plg_info(ctx->ins, "db: delete unmonitored stale inodes from the database: count=%d", - changes); - if (db_locked == FLB_TRUE) { - tail_db_unlock(ctx); - } - - return 0; - } - - /* Create/clear temp table holding current monitored inodes. */ - ret = flb_sqldb_query(ctx->db, - "CREATE TEMP TABLE IF NOT EXISTS in_tail_current_inodes (" - " inode INTEGER PRIMARY KEY" - ");", - NULL, NULL); - if (ret != FLB_OK) { - flb_plg_error(ctx->ins, "db: cannot create temp table for inode cleanup"); - goto error; - } - - ret = flb_sqldb_query(ctx->db, "DELETE FROM in_tail_current_inodes;", NULL, NULL); - if (ret != FLB_OK) { - flb_plg_error(ctx->ins, "db: cannot clear temp inode table"); - goto error; - } - - /* Use a transaction for faster bulk inserts. */ - ret = flb_sqldb_query(ctx->db, "BEGIN;", NULL, NULL); - if (ret != FLB_OK) { - flb_plg_error(ctx->ins, "db: cannot begin transaction for temp inode inserts"); - goto error; - } - txn_started = FLB_TRUE; - - ret = sqlite3_prepare_v2(ctx->db->handler, - "INSERT OR IGNORE INTO in_tail_current_inodes(inode) VALUES (?);", - -1, &stmt_insert_inode, 0); - if (ret != SQLITE_OK) { - flb_plg_error(ctx->ins, "db: cannot prepare temp inode insert statement, ret=%d", ret); - goto error; - } - - mk_list_foreach_safe(head, tmp, &ctx->files_static) { - file = mk_list_entry(head, struct flb_tail_file, _head); - - ret = sqlite3_bind_int64(stmt_insert_inode, 1, (sqlite3_int64) file->inode); - if (ret != SQLITE_OK) { - flb_plg_error(ctx->ins, "db: error binding temp inode insert: inode=%" PRIu64 ", ret=%d", - file->inode, ret); - goto error; - } - - ret = sqlite3_step(stmt_insert_inode); - if (ret != SQLITE_DONE) { - flb_plg_error(ctx->ins, "db: error inserting inode into temp table: inode=%" PRIu64 ", ret=%d", - file->inode, ret); - goto error; - } - - sqlite3_clear_bindings(stmt_insert_inode); - sqlite3_reset(stmt_insert_inode); - } - - sqlite3_finalize(stmt_insert_inode); - stmt_insert_inode = NULL; - - /* Delete any inode that is not in the current monitored set. */ - ret = flb_sqldb_query(ctx->db, - "DELETE FROM in_tail_files " - "WHERE inode NOT IN (SELECT inode FROM in_tail_current_inodes);", - NULL, NULL); - if (ret != FLB_OK) { - flb_plg_error(ctx->ins, "db: cannot delete stale inodes using temp table"); - goto error; - } - - ret = flb_sqldb_query(ctx->db, "COMMIT;", NULL, NULL); - if (ret != FLB_OK) { - flb_plg_error(ctx->ins, "db: cannot commit transaction for temp inode inserts"); - goto error; - } - txn_started = FLB_FALSE; - - changes = sqlite3_changes(ctx->db->handler); - flb_plg_info(ctx->ins, "db: delete unmonitored stale inodes from the database: count=%d", - changes); - - if (db_locked == FLB_TRUE) { - tail_db_unlock(ctx); - } - - return 0; - -error: - if (stmt_insert_inode) { - sqlite3_finalize(stmt_insert_inode); - } - - if (txn_started == FLB_TRUE) { - /* Best-effort rollback */ - flb_sqldb_query(ctx->db, "ROLLBACK;", NULL, NULL); - } - - if (db_locked == FLB_TRUE) { - tail_db_unlock(ctx); - } - - return -1; -} - int flb_tail_db_file_set(struct flb_tail_file *file, struct flb_tail_config *ctx) { @@ -639,199 +569,56 @@ int flb_tail_db_file_delete(struct flb_tail_file *file, return 0; } -/* - * Delete stale file from database - */ -int flb_tail_db_stale_file_delete(struct flb_input_instance *ins, - struct flb_config *config, - struct flb_tail_config *ctx) +int flb_tail_db_cleanup(struct flb_tail_config *ctx, + flb_tail_db_inode_check_fn inode_is_monitored, + void *data) { - int ret = -1; - size_t sql_size; - uint64_t idx; - uint64_t file_count = ctx->files_static_count; - int max_vars = -1; - flb_sds_t stale_delete_sql; - flb_sds_t sds_tmp; - sqlite3_stmt *stmt_delete_inodes = NULL; - struct mk_list *tmp; - struct mk_list *head; - struct flb_tail_file *file; - int db_locked = FLB_FALSE; + int ret; + int deleted_count; + sqlite3_stmt *stmt_stale_files = NULL; - if (!ctx->db) { + if (ctx->db == NULL) { return 0; } - ret = tail_db_lock(ctx); - if (ret != 0) { - flb_plg_error(ctx->ins, "db: could not acquire lock"); - return -1; - } - - db_locked = FLB_TRUE; - /* - * Avoid SQLite variable limits for large monitored file sets. - * - * sqlite3_limit(..., SQLITE_LIMIT_VARIABLE_NUMBER, -1) returns the current - * runtime limit (compile-time hard limit may be higher). If our monitored - * file count exceeds this, the legacy NOT IN (?,?,...) statement will fail - * at prepare-time. + * Only the original database context performs maintenance once for the + * shared SQLite handler. */ - max_vars = sqlite3_limit(ctx->db->handler, SQLITE_LIMIT_VARIABLE_NUMBER, -1); - if (max_vars > 0 && file_count > (uint64_t) max_vars) { - flb_plg_warn(ctx->ins, - "db: large file set detected (%" PRIu64 " files) exceeds SQLite variable limit (%d); " - "using temp-table cleanup for stale inode deletion", - file_count, max_vars); - return flb_tail_db_stale_file_delete_temp_table(ctx, file_count, db_locked); - } - - /* Create a stmt sql buffer */ - sql_size = SQL_DELETE_STALE_FILE_START_LEN; - sql_size += SQL_DELETE_STALE_FILE_WHERE_LEN; - sql_size += SQL_STMT_START_PARAM_LEN; - sql_size += SQL_STMT_PARAM_END_LEN; - sql_size += SQL_STMT_END_LEN; - if (file_count > 0) { - /* - * We already account for the first '?' via SQL_STMT_START_PARAM_LEN. - * Additional parameters are count-1 occurrences of ",?". - */ - if (file_count > 1) { - sql_size += (SQL_STMT_ADD_PARAM_LEN * (file_count - 1)); - } - } - - stale_delete_sql = flb_sds_create_size(sql_size + 1); - if (!stale_delete_sql) { - flb_plg_error(ctx->ins, "cannot allocate buffer for stale_delete_sql:" - " size: %zu", sql_size); - if (db_locked == FLB_TRUE) { - tail_db_unlock(ctx); - } - - return -1; - } - - /* Create a stmt sql */ - sds_tmp = flb_sds_cat(stale_delete_sql, SQL_DELETE_STALE_FILE_START, - SQL_DELETE_STALE_FILE_START_LEN); - if (sds_tmp == NULL) { - flb_plg_error(ctx->ins, - "error concatenating stale_delete_sql: start"); - flb_sds_destroy(stale_delete_sql); - if (db_locked == FLB_TRUE) { - tail_db_unlock(ctx); - } - - return -1; - } - stale_delete_sql = sds_tmp; - - if (file_count > 0) { - sds_tmp = flb_sds_cat(stale_delete_sql, SQL_DELETE_STALE_FILE_WHERE, - SQL_DELETE_STALE_FILE_WHERE_LEN); - if (sds_tmp == NULL) { - flb_plg_error(ctx->ins, - "error concatenating stale_delete_sql: where"); - flb_sds_destroy(stale_delete_sql); - if (db_locked == FLB_TRUE) { - tail_db_unlock(ctx); - } - - return -1; - } - stale_delete_sql = sds_tmp; - - ret = stmt_add_param_concat(ctx, &stale_delete_sql, file_count); - if (ret == -1) { - flb_plg_error(ctx->ins, - "error concatenating stale_delete_sql: param"); - flb_sds_destroy(stale_delete_sql); - if (db_locked == FLB_TRUE) { - tail_db_unlock(ctx); - } - - return -1; - } + if (ctx->db->shared == FLB_TRUE) { + return 0; } - sds_tmp = flb_sds_cat(stale_delete_sql, SQL_STMT_END, SQL_STMT_END_LEN); - if (sds_tmp == NULL) { - flb_plg_error(ctx->ins, - "error concatenating stale_delete_sql: end"); - flb_sds_destroy(stale_delete_sql); - if (db_locked == FLB_TRUE) { - tail_db_unlock(ctx); - } - + ret = tail_db_lock(ctx); + if (ret != 0) { + flb_plg_error(ctx->ins, "db: could not acquire lock"); return -1; } - stale_delete_sql = sds_tmp; - /* Prepare stmt */ - ret = sqlite3_prepare_v2(ctx->db->handler, stale_delete_sql, -1, - &stmt_delete_inodes, 0); + ret = sqlite3_prepare_v2(ctx->db->handler, SQL_SELECT_STALE_FILES, -1, + &stmt_stale_files, 0); if (ret != SQLITE_OK) { - flb_plg_error(ctx->ins, "error preparing database SQL statement:" - " stmt_delete_inodes sql:%s, ret=%d", stale_delete_sql, - ret); - flb_sds_destroy(stale_delete_sql); - if (db_locked == FLB_TRUE) { - tail_db_unlock(ctx); - } - - return -1; - } - - /* Bind parameters */ - idx = 1; - mk_list_foreach_safe(head, tmp, &ctx->files_static) { - file = mk_list_entry(head, struct flb_tail_file, _head); - ret = sqlite3_bind_int64(stmt_delete_inodes, idx, file->inode); - if (ret != SQLITE_OK) { - flb_plg_error(ctx->ins, "error binding to stmt_delete_inodes:" - " inode=%" PRIu64 ", ret=%d", file->inode, ret); - sqlite3_finalize(stmt_delete_inodes); - flb_sds_destroy(stale_delete_sql); - if (db_locked == FLB_TRUE) { - tail_db_unlock(ctx); - } - - return -1; - } - idx++; + flb_plg_error(ctx->ins, "db: cannot prepare stale file query: ret=%d", ret); + goto error; } - /* Run the delete inodes */ - ret = sqlite3_step(stmt_delete_inodes); - if (ret != SQLITE_DONE) { - sqlite3_finalize(stmt_delete_inodes); - flb_sds_destroy(stale_delete_sql); - flb_plg_error(ctx->ins, "cannot execute delete stale inodes: ret=%d", - ret); - - if (db_locked == FLB_TRUE) { - tail_db_unlock(ctx); - } - - return -1; + ret = stale_file_delete_missing(ctx, inode_is_monitored, data, + stmt_stale_files, &deleted_count); + sqlite3_finalize(stmt_stale_files); + stmt_stale_files = NULL; + if (ret != 0) { + goto error; } - ret = sqlite3_changes(ctx->db->handler); - flb_plg_info(ctx->ins, "db: delete unmonitored stale inodes from the" - " database: count=%d", ret); - - sqlite3_finalize(stmt_delete_inodes); - flb_sds_destroy(stale_delete_sql); + flb_plg_info(ctx->ins, "db: cleaned stale file records: count=%d", + deleted_count); + tail_db_unlock(ctx); + return 0; - if (db_locked == FLB_TRUE) { - tail_db_unlock(ctx); +error: + if (stmt_stale_files != NULL) { + sqlite3_finalize(stmt_stale_files); } - - db_locked = FLB_FALSE; - - return 0; + tail_db_unlock(ctx); + return -1; } diff --git a/plugins/in_tail/tail_db.h b/plugins/in_tail/tail_db.h index 6b78cf0ce5e..5e95a9b35df 100644 --- a/plugins/in_tail/tail_db.h +++ b/plugins/in_tail/tail_db.h @@ -25,6 +25,8 @@ #include "tail_file.h" +typedef int (*flb_tail_db_inode_check_fn)(uint64_t inode, void *data); + struct flb_sqldb *flb_tail_db_open(const char *path, struct flb_input_instance *in, struct flb_tail_config *ctx, @@ -40,7 +42,7 @@ int flb_tail_db_file_rotate(const char *new_name, struct flb_tail_config *ctx); int flb_tail_db_file_delete(struct flb_tail_file *file, struct flb_tail_config *ctx); -int flb_tail_db_stale_file_delete(struct flb_input_instance *ins, - struct flb_config *config, - struct flb_tail_config *ctx); +int flb_tail_db_cleanup(struct flb_tail_config *ctx, + flb_tail_db_inode_check_fn inode_is_monitored, + void *data); #endif diff --git a/plugins/in_tail/tail_sql.h b/plugins/in_tail/tail_sql.h index 6c2868a6ede..8b37402c8ae 100644 --- a/plugins/in_tail/tail_sql.h +++ b/plugins/in_tail/tail_sql.h @@ -65,27 +65,8 @@ #define SQL_DELETE_FILE \ "DELETE FROM in_tail_files WHERE id=@id;" -#define SQL_STMT_START_PARAM "(?" -#define SQL_STMT_START_PARAM_LEN (sizeof(SQL_STMT_START_PARAM) - 1) - -#define SQL_STMT_ADD_PARAM ",?" -#define SQL_STMT_ADD_PARAM_LEN (sizeof(SQL_STMT_ADD_PARAM) - 1) - -#define SQL_STMT_PARAM_END ")" -#define SQL_STMT_PARAM_END_LEN (sizeof(SQL_STMT_PARAM_END) - 1) - -#define SQL_STMT_END ";" -#define SQL_STMT_END_LEN (sizeof(SQL_STMT_END) - 1) - -#define SQL_DELETE_STALE_FILE_START \ - "DELETE FROM in_tail_files " -#define SQL_DELETE_STALE_FILE_START_LEN \ - (sizeof(SQL_DELETE_STALE_FILE_START) - 1) - -#define SQL_DELETE_STALE_FILE_WHERE \ - "WHERE inode NOT IN " -#define SQL_DELETE_STALE_FILE_WHERE_LEN \ - (sizeof(SQL_DELETE_STALE_FILE_WHERE) - 1) +#define SQL_SELECT_STALE_FILES \ + "SELECT id, name, inode FROM in_tail_files;" #define SQL_PRAGMA_SYNC \ "PRAGMA synchronous=%i;" diff --git a/tests/runtime/in_tail.c b/tests/runtime/in_tail.c index 20073afe811..54a81df55fc 100644 --- a/tests/runtime/in_tail.c +++ b/tests/runtime/in_tail.c @@ -29,6 +29,9 @@ Approach for this tests is basing on filter_kubernetes tests #ifdef FLB_HAVE_UNICODE_ENCODER #include #endif +#ifdef FLB_HAVE_SQLDB +#include +#endif #include #include #include @@ -2917,6 +2920,374 @@ void flb_test_db_delete_stale_file() unlink(db); } +static int tail_db_contains_file(const char *db_path, const char *file_path, + uint64_t inode) +{ + int found = FLB_FALSE; + int ret; + const unsigned char *name; + sqlite3 *db = NULL; + sqlite3_stmt *stmt = NULL; + + ret = sqlite3_open(db_path, &db); + if (ret != SQLITE_OK) { + goto cleanup; + } + + ret = sqlite3_prepare_v2(db, "SELECT name, inode FROM in_tail_files;", + -1, &stmt, NULL); + if (ret != SQLITE_OK) { + goto cleanup; + } + + ret = sqlite3_step(stmt); + if (ret != SQLITE_ROW) { + goto cleanup; + } + + name = sqlite3_column_text(stmt, 0); + if (name == NULL || strcmp((const char *) name, file_path) != 0 || + sqlite3_column_int64(stmt, 1) != (sqlite3_int64) inode) { + goto cleanup; + } + + ret = sqlite3_step(stmt); + if (ret == SQLITE_DONE) { + found = FLB_TRUE; + } + +cleanup: + if (stmt != NULL) { + sqlite3_finalize(stmt); + } + if (db != NULL) { + sqlite3_close(db); + } + + return found; +} + +void flb_test_db_replaced_file_cleanup() +{ + struct flb_lib_out_cb cb_data; + struct test_tail_ctx *ctx; + char *file[] = {"test_db_replaced.log"}; + char *old_file = "test_db_replaced_old.log"; + char *db = "test_db_replaced.db"; + char *old_message = "old message"; + char *new_message = "new message"; + struct stat st; + int unused; + int ret; + int num; + + unlink(db); + unlink(old_file); + clear_output_num(); + + cb_data.cb = cb_count_msgpack; + cb_data.data = &unused; + + ctx = test_tail_ctx_create(&cb_data, file, 1, FLB_TRUE); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("test_ctx_create failed"); + exit(EXIT_FAILURE); + } + + ret = flb_input_set(ctx->flb, ctx->i_ffd, + "path", file[0], + "read_from_head", "true", + "db", db, + "db.sync", "full", + NULL); + TEST_CHECK(ret == 0); + + ret = flb_output_set(ctx->flb, ctx->o_ffd, NULL); + TEST_CHECK(ret == 0); + + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + ret = write_msg(ctx, old_message, strlen(old_message)); + TEST_CHECK(ret > 0); + flb_time_msleep(500); + + close(ctx->fds[0]); + flb_free(ctx->fds); + flb_stop(ctx->flb); + flb_destroy(ctx->flb); + flb_free(ctx); + + ret = rename(file[0], old_file); + TEST_CHECK(ret == 0); + clear_output_num(); + + ctx = test_tail_ctx_create(&cb_data, file, 1, FLB_TRUE); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("test_ctx_create failed"); + unlink(old_file); + unlink(db); + exit(EXIT_FAILURE); + } + + ret = write_msg(ctx, new_message, strlen(new_message)); + TEST_CHECK(ret > 0); + + ret = flb_input_set(ctx->flb, ctx->i_ffd, + "path", file[0], + "read_from_head", "true", + "db", db, + "db.sync", "full", + NULL); + TEST_CHECK(ret == 0); + + ret = flb_output_set(ctx->flb, ctx->o_ffd, NULL); + TEST_CHECK(ret == 0); + + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + flb_time_msleep(500); + + num = get_output_num(); + if (!TEST_CHECK(num == 1)) { + TEST_MSG("num error. expect=1 got=%d", num); + } + + ret = stat(file[0], &st); + TEST_CHECK(ret == 0); + + ret = tail_db_contains_file(db, file[0], st.st_ino); + if (!TEST_CHECK(ret == FLB_TRUE)) { + TEST_MSG("database does not contain only the replacement file"); + } + + test_tail_ctx_destroy(ctx); + unlink(old_file); + unlink(db); +} + +void flb_test_db_renamed_file_preserves_offset() +{ + struct flb_lib_out_cb cb_data; + struct test_tail_ctx *ctx; + char *file[] = {"test_db_renamed.log"}; + char *renamed_file[] = {"test_db_renamed.log.1"}; + char *db = "test_db_renamed.db"; + char *old_message = "old message"; + char *new_message = "new message"; + int unused; + int ret; + int num; + + unlink(db); + unlink(renamed_file[0]); + clear_output_num(); + + cb_data.cb = cb_count_msgpack; + cb_data.data = &unused; + + ctx = test_tail_ctx_create(&cb_data, file, 1, FLB_TRUE); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("test_ctx_create failed"); + exit(EXIT_FAILURE); + } + + ret = flb_input_set(ctx->flb, ctx->i_ffd, + "path", file[0], + "read_from_head", "true", + "db", db, + "db.sync", "full", + NULL); + TEST_CHECK(ret == 0); + + ret = flb_output_set(ctx->flb, ctx->o_ffd, NULL); + TEST_CHECK(ret == 0); + + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + ret = write_msg(ctx, old_message, strlen(old_message)); + TEST_CHECK(ret > 0); + flb_time_msleep(500); + + close(ctx->fds[0]); + flb_free(ctx->fds); + flb_stop(ctx->flb); + flb_destroy(ctx->flb); + flb_free(ctx); + + ret = rename(file[0], renamed_file[0]); + TEST_CHECK(ret == 0); + clear_output_num(); + + ctx = test_tail_ctx_create(&cb_data, renamed_file, 1, FLB_FALSE); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("test_ctx_create failed"); + unlink(renamed_file[0]); + unlink(db); + exit(EXIT_FAILURE); + } + + ret = flb_input_set(ctx->flb, ctx->i_ffd, + "path", renamed_file[0], + "read_from_head", "true", + "db", db, + "db.sync", "full", + NULL); + TEST_CHECK(ret == 0); + + ret = flb_output_set(ctx->flb, ctx->o_ffd, NULL); + TEST_CHECK(ret == 0); + + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + flb_time_msleep(500); + + num = get_output_num(); + if (!TEST_CHECK(num == 0)) { + TEST_MSG("num error after rename. expect=0 got=%d", num); + } + + ret = write_msg(ctx, new_message, strlen(new_message)); + TEST_CHECK(ret > 0); + flb_time_msleep(500); + + num = get_output_num(); + if (!TEST_CHECK(num == 1)) { + TEST_MSG("num error after append. expect=1 got=%d", num); + } + + test_tail_ctx_destroy(ctx); + unlink(db); +} + +void flb_test_db_shared_between_inputs() +{ + struct flb_lib_out_cb cb_data; + struct test_tail_ctx *ctx; + char *files[] = {"test_db_shared_a.log", "test_db_shared_b.log"}; + char *db = "test_db_shared.db"; + char *msg_init = "initial message"; + char *msg_end = "new message"; + int second_input; + int unused; + int ret; + int num; + int i; + + unlink(db); + clear_output_num(); + + cb_data.cb = cb_count_msgpack; + cb_data.data = &unused; + + ctx = test_tail_ctx_create(&cb_data, files, + sizeof(files) / sizeof(char *), FLB_TRUE); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("test_ctx_create failed"); + exit(EXIT_FAILURE); + } + + second_input = flb_input(ctx->flb, "tail", NULL); + TEST_CHECK(second_input >= 0); + + ret = flb_input_set(ctx->flb, ctx->i_ffd, + "tag", "shared_db", + "path", files[0], + "read_from_head", "true", + "db", db, + "db.sync", "full", + NULL); + TEST_CHECK(ret == 0); + + ret = flb_input_set(ctx->flb, second_input, + "tag", "shared_db", + "path", files[1], + "read_from_head", "true", + "db", db, + "db.sync", "full", + NULL); + TEST_CHECK(ret == 0); + + ret = flb_output_set(ctx->flb, ctx->o_ffd, + "match", "shared_db", + NULL); + TEST_CHECK(ret == 0); + + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + ret = write_msg(ctx, msg_init, strlen(msg_init)); + TEST_CHECK(ret > 0); + + flb_time_msleep(500); + num = get_output_num(); + if (!TEST_CHECK(num == 2)) { + TEST_MSG("num error. expect=2 got=%d", num); + } + + for (i = 0; i < ctx->fd_num; i++) { + close(ctx->fds[i]); + } + flb_free(ctx->fds); + flb_stop(ctx->flb); + flb_destroy(ctx->flb); + flb_free(ctx); + + clear_output_num(); + + ctx = test_tail_ctx_create(&cb_data, files, + sizeof(files) / sizeof(char *), FLB_FALSE); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("test_ctx_create failed"); + unlink(db); + exit(EXIT_FAILURE); + } + + second_input = flb_input(ctx->flb, "tail", NULL); + TEST_CHECK(second_input >= 0); + + ret = flb_input_set(ctx->flb, ctx->i_ffd, + "tag", "shared_db", + "path", files[0], + "read_from_head", "true", + "db", db, + "db.sync", "full", + NULL); + TEST_CHECK(ret == 0); + + ret = flb_input_set(ctx->flb, second_input, + "tag", "shared_db", + "path", files[1], + "read_from_head", "true", + "db", db, + "db.sync", "full", + NULL); + TEST_CHECK(ret == 0); + + ret = flb_output_set(ctx->flb, ctx->o_ffd, + "match", "shared_db", + NULL); + TEST_CHECK(ret == 0); + + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + flb_time_msleep(500); + + ret = write_msg(ctx, msg_end, strlen(msg_end)); + TEST_CHECK(ret > 0); + + flb_time_msleep(500); + num = get_output_num(); + if (!TEST_CHECK(num == 2)) { + TEST_MSG("num error. expect=2 got=%d", num); + } + + test_tail_ctx_destroy(ctx); + unlink(db); +} + void flb_test_db_compare_filename() { struct flb_lib_out_cb cb_data; @@ -3100,6 +3471,9 @@ TEST_LIST = { #ifdef FLB_HAVE_SQLDB {"db", flb_test_db}, {"db_delete_stale_file", flb_test_db_delete_stale_file}, + {"db_replaced_file_cleanup", flb_test_db_replaced_file_cleanup}, + {"db_renamed_file_preserves_offset", flb_test_db_renamed_file_preserves_offset}, + {"db_shared_between_inputs", flb_test_db_shared_between_inputs}, {"db_compare_filename", flb_test_db_compare_filename}, #endif