From dd653fb84ee65e0d0b7f56a44529327599cc741e Mon Sep 17 00:00:00 2001 From: Siddarth Raj Date: Tue, 2 Jun 2026 02:02:09 +0530 Subject: [PATCH] DAOS-18922: Unit test improvements for DAOS placement algorithm Signed-off-by: Siddarth Raj --- src/placement/tests/SConscript | 8 +- src/placement/tests/layout_test.c | 740 +++++++++++++++++++ src/placement/tests/layout_test_helpers.c | 797 +++++++++++++++++++++ src/placement/tests/layout_test_helpers.h | 102 +++ src/tests/ftest/SConscript | 2 +- src/tests/ftest/placement/layout_test.py | 236 ++++++ src/tests/ftest/placement/layout_test.yaml | 126 ++++ 7 files changed, 2009 insertions(+), 2 deletions(-) create mode 100644 src/placement/tests/layout_test.c create mode 100644 src/placement/tests/layout_test_helpers.c create mode 100644 src/placement/tests/layout_test_helpers.h create mode 100644 src/tests/ftest/placement/layout_test.py create mode 100644 src/tests/ftest/placement/layout_test.yaml diff --git a/src/placement/tests/SConscript b/src/placement/tests/SConscript index b60893f268f..599ca82f025 100644 --- a/src/placement/tests/SConscript +++ b/src/placement/tests/SConscript @@ -25,19 +25,25 @@ def scons(): 'jump_map_dist.c', 'placement_test.c']) pl_bench_tgt = denv.SharedObject(['pl_bench.c', 'place_obj_common.c']) + layout_test_tgt = denv.SharedObject(['layout_test.c', + 'layout_test_helpers.c', + 'place_obj_common.c']) + srv_pool_map_tgt = denv.SharedObject('placement_srv_pool_map', '../../pool/srv_pool_map.c') libraries = ['daos', 'daos_common', 'gurt', 'uuid', 'cmocka', 'isal', 'm'] ring_pl_test = denv.d_program('ring_pl_map', ring_test_tgt, LIBS=libraries) jump_pl_test = denv.d_program('jump_pl_map', - jump_test_tgt + ['../../pool/srv_pool_map.c'], LIBS=libraries) + jump_test_tgt + srv_pool_map_tgt, LIBS=libraries) pl_bench = denv.d_program('pl_bench', pl_bench_tgt, LIBS=libraries) + layout_test = denv.d_program('layout_test', layout_test_tgt + srv_pool_map_tgt, LIBS=libraries) denv.Install('$PREFIX/bin/', ring_pl_test) denv.Install('$PREFIX/bin/', jump_pl_test) denv.Install('$PREFIX/bin/', pl_bench) + denv.Install('$PREFIX/bin/', layout_test) if __name__ == "SCons.Script": diff --git a/src/placement/tests/layout_test.c b/src/placement/tests/layout_test.c new file mode 100644 index 00000000000..a7eefe09817 --- /dev/null +++ b/src/placement/tests/layout_test.c @@ -0,0 +1,740 @@ +/** + * (C) Copyright 2026 Hewlett Packard Enterprise Development LP + * + * SPDX-License-Identifier: BSD-2-Clause-Patent + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "place_obj_common.h" +#include "layout_test_helpers.h" + +static void +print_usage(const char *prog) +{ + printf("Usage: %s [options]\n", prog); + printf("\nOptions:\n"); + printf(" -n, --nodes N Number of nodes (default: %d)\n", DEFAULT_NODES); + printf(" -r, --ranks N Ranks per node (default: %d)\n", DEFAULT_RANKS); + printf(" -t, --targets N Targets per rank (default: %d)\n", DEFAULT_TARGETS); + printf(" -c, --class CLASS Object class (default: %s)\n", DEFAULT_OBJ_CLASS); + printf(" -N, --obj-count N Number of OIDs (default: %d)\n", DEFAULT_OBJ_COUNT); + printf(" -o, --operation STR Comma-separated list of operations\n"); + printf(" Format: =[]\n"); + printf(" Types : exclude, reint, drain, add\n"); + printf(" Modes : rank, node, target\n"); + printf(" Example: \"exclude rank=[0,1], reint node=[2]\"\n"); + printf(" -h, --help Print this help message\n"); +} + +static int +parse_int_arg(const char *name, const char *value, int64_t *out) +{ + char *end = NULL; + long long parsed; + + errno = 0; + parsed = strtoll(value, &end, 10); + if (errno != 0 || end == value || *end != '\0') { + fprintf(stderr, "ERROR: %s must be an integer, got \"%s\"\n", name, value); + *out = -1; + return -DER_INVAL; + } + + *out = parsed; + return 0; +} + +static int +get_oclass_ft(daos_oclass_id_t oclass) +{ + struct daos_oclass_attr *attr; + attr = daos_oclass_id2attr(oclass, 0); + switch (attr->ca_resil) { + case DAOS_RES_REPL: + return attr->u.rp.r_num - 1; + case DAOS_RES_EC: + return attr->u.ec.e_p; + default: + return 0; + } +} + +static void +format_memory_value(uint64_t bytes, double *value, const char **unit) +{ + const uint64_t kb = 1024ULL; + const uint64_t mb = kb * kb; + const uint64_t gb = mb * kb; + + if (bytes >= gb) { + *value = (double)bytes / (double)gb; + *unit = "GB"; + return; + } + + if (bytes >= mb) { + *value = (double)bytes / (double)mb; + *unit = "MB"; + return; + } + + *value = (double)bytes; + *unit = "bytes"; +} + +static void +print_memory_line(const char *label, uint64_t bytes) +{ + double value; + const char *unit; + + format_memory_value(bytes, &value, &unit); + if (strcmp(unit, "bytes") == 0) + printf(" %-20s: %llu bytes\n", label, (unsigned long long)bytes); + else + printf(" %-20s: %.2f %s\n", label, value, unit); +} + +static void +print_time_line(const char *label, double ms) +{ + if (ms >= 3600000.0) + printf(" %-28s: %.3f h\n", label, ms / 3600000.0); + else if (ms >= 60000.0) + printf(" %-28s: %.3f min\n", label, ms / 60000.0); + else if (ms >= 1000.0) + printf(" %-28s: %.3f s\n", label, ms / 1000.0); + else + printf(" %-28s: %.3f ms\n", label, ms); +} + +/* + * Parse a comma-separated list of operations from a string of the form + * " =[], ...". Each token is split on commas that are + * not inside square brackets so that multi-value arguments such as + * node=[1,2] are kept intact. The results are written into ops[] and + * op_count is set to the number of operations parsed. + * + * Returns 0 on success, -1 on parse error. + */ +static int +parse_operations(char *input, struct operation ops[], int *op_count) +{ + char *pos = input; + *op_count = 0; + + while (pos != NULL && *pos != '\0') { + char type_str[32]; + char mode_str[32]; + char arg_str[128]; + char *arg; + char *arg_saveptr = NULL; + struct operation *op; + char *token; + char *p; + int depth; + bool found_sep; + + if (*op_count >= MAX_OPERATIONS) { + fprintf(stderr, "ERROR: too many operations (max %d)\n", MAX_OPERATIONS); + return -1; + } + + while (*pos == ' ') + pos++; + + if (*pos == '\0') + break; + + token = pos; + depth = 0; + found_sep = false; + + /* + * Advance pos to the next comma that is NOT inside [...], + * so that multi-value args like node=[1,2] are kept intact. + */ + for (p = pos; *p != '\0'; p++) { + if (*p == '[') + depth++; + else if (*p == ']') + depth--; + else if (*p == ',' && depth == 0) { + *p = '\0'; + pos = p + 1; + found_sep = true; + break; + } + } + if (!found_sep) + pos = NULL; /* last token, no more after this */ + + { + int consumed = 0; + + if (sscanf(token, "%31s %31[^=]=[%127[^]]%n", type_str, mode_str, arg_str, + &consumed) != 3) { + fprintf(stderr, + "ERROR: invalid operation format: \"%s\"\n" + " Expected: =[]\n" + " Example : exclude node=[0,1]\n", + token); + return -1; + } + + /* Skip the closing ']' that sscanf left in the stream */ + const char *leftover = token + consumed; + + if (*leftover == ']') + leftover++; + while (*leftover == ' ') + leftover++; + if (*leftover != '\0') { + fprintf(stderr, + "ERROR: operations must be comma-separated, got " + "space-separated " + "extra content: \"%s\"\n" + " Use commas between operations, e.g.: " + "\"exclude node=[2], add node=[8]\"\n", + leftover); + return -1; + } + } + + op = &ops[*op_count]; + + /* type */ + if (strcmp(type_str, "exclude") == 0) + op->type = OP_EXCLUDE; + else if (strcmp(type_str, "reint") == 0) + op->type = OP_REINT; + else if (strcmp(type_str, "drain") == 0) + op->type = OP_DRAIN; + else if (strcmp(type_str, "add") == 0) + op->type = OP_ADD; + else { + fprintf(stderr, + "ERROR: unknown operation type: \"%s\"\n" + " Valid types: exclude, reint, drain, add\n", + type_str); + return -1; + } + + /* component */ + if (strcmp(mode_str, "rank") == 0) + op->component = RANK; + else if (strcmp(mode_str, "node") == 0) + op->component = NODE; + else if (strcmp(mode_str, "target") == 0) + op->component = TARGET; + else { + fprintf(stderr, + "ERROR: unknown operation component: \"%s\"\n" + " Valid components: rank, node, target\n", + mode_str); + return -1; + } + + /* cross-validate type + component */ + if (op->type == OP_ADD && op->component != NODE) { + fprintf(stderr, + "ERROR: \"add\" only supports node, got \"%s\"\n" + " Usage: add node=[]\n", + mode_str); + return -1; + } + + /* args */ + op->nr_args = 0; + + arg = strtok_r(arg_str, ",", &arg_saveptr); + + while (arg != NULL) { + char *end = NULL; + long val; + size_t len; + + while (*arg == ' ') + arg++; + len = strlen(arg); + while (len > 0 && arg[len - 1] == ' ') { + arg[len - 1] = '\0'; + len--; + } + + if (op->nr_args >= MAX_OP_ARGS) + return -1; + + errno = 0; + val = strtol(arg, &end, 10); + if (errno != 0 || end == arg || *end != '\0' || val < INT_MIN || + val > INT_MAX) { + fprintf(stderr, + "ERROR: invalid operation argument \"%s\"; " + "must be an integer\n", + arg); + return -1; + } + + op->args[op->nr_args++] = (int)val; + + arg = strtok_r(NULL, ",", &arg_saveptr); + } + + (*op_count)++; + } + return 0; +} + +static const char * +op_type_str(enum operation_type type) +{ + switch (type) { + case OP_EXCLUDE: + return "exclude"; + case OP_REINT: + return "reint"; + case OP_DRAIN: + return "drain"; + case OP_ADD: + return "add"; + default: + return "unknown"; + } +} + +static const char * +op_component_str(enum operation_component component) +{ + switch (component) { + case RANK: + return "rank"; + case NODE: + return "node"; + case TARGET: + return "target"; + default: + return "unknown"; + } +} + +/* + * Execute all operations in sequence. For each operation: + * 1. Capture the initial layout (pre-operation). + * 2. For OP_ADD only: insert the new node(s) into the pool map. + * 3. Build the complete target list (by target ID, rank, or node). + * 4. Apply the two-stage status change and collect the per-OID rebuild diff. + * 5. Capture the post-operation layout. + * 6. Validate the diff against the layout delta; print moved-shard + * percentage, performance (pl_obj_place / pl_obj_find_rebuild wall-clock), + * and per-operation memory usage. + * + * @param ctx Test context (pool/placement maps, OID count, oclass) + * @param operations Array of operations to execute + * @param operation_count Number of operations + * @return 0 on success, negative DAOS error code on failure + */ +static int +execute_tests(struct test_ctx *ctx, struct operation *operations, int operation_count) +{ + int i; + int rc = 0; + + for (i = 0; i < operation_count; i++) { + /* + * For OP_ADD/extend the jump map rebalances freely across the + * new topology — any number of shards per group can move, so + * Check 1 (bounded drift) does not apply. Pass INT_MAX to + * effectively skip it. Check 2 (diff exactness) still runs. + */ + int max_diff = + (operations[i].type == OP_ADD) ? INT_MAX : get_oclass_ft(ctx->oclass); + struct pl_obj_layout **orig_layout = NULL; + struct pl_obj_layout **post_layout = NULL; + struct shard_diff *diff = NULL; + struct pool_target_id_list tgts = {0}; + int a; + uint64_t op_layout_test_mem = 0; + uint64_t op_daos_api_mem = 0; + + printf("\n[%d/%d] Running %s %s operation\n", i + 1, operation_count, + op_type_str(operations[i].type), op_component_str(operations[i].component)); + + do { + D_ALLOC_ARRAY(orig_layout, ctx->num_oids); + if (orig_layout == NULL) { + rc = -DER_NOMEM; + break; + } + op_layout_test_mem += sizeof(*orig_layout) * ctx->num_oids; + + D_ALLOC_ARRAY(post_layout, ctx->num_oids); + if (post_layout == NULL) { + rc = -DER_NOMEM; + break; + } + op_layout_test_mem += sizeof(*post_layout) * ctx->num_oids; + + D_ALLOC_ARRAY(diff, ctx->num_oids); + if (diff == NULL) { + rc = -DER_NOMEM; + break; + } + op_layout_test_mem += sizeof(*diff) * ctx->num_oids; + + double ms_place1 = 0.0; + double ms_rebuild = 0.0; + double ms_place2 = 0.0; + + /* Capture the initial layout before the operation */ + printf(" Capturing initial layouts...\n"); + rc = capture_layouts(ctx, orig_layout, &ms_place1); + if (rc != 0) { + D_ERROR("capture_layouts failed rc=%d\n", rc); + break; + } + /* Count actual DAOS API allocation: struct header + ol_shards array */ + { + int k; + + for (k = 0; k < ctx->num_oids; k++) { + if (orig_layout[k] != NULL) + op_daos_api_mem += + sizeof(*orig_layout[k]) + + orig_layout[k]->ol_nr * + sizeof(*orig_layout[k]->ol_shards); + } + } + + /* + * For OP_ADD, insert the new node(s) into the pool map + * before trying to fetch their targets. + */ + if (operations[i].type == OP_ADD) { + for (a = 0; a < operations[i].nr_args; a++) { + int new_comps; + + rc = add_node(ctx, operations[i].args[a]); + if (rc != 0) { + D_ERROR("add_node failed rc=%d\n", rc); + break; + } + /* + * add_node allocates a comps array then a pool_buf; + * both are freed before it returns, but they represent + * peak memory during the call. Account for them here. + * pool_map_extend internals are opaque and not tracked. + */ + new_comps = 1 + ctx->ranks_per_node + + ctx->ranks_per_node * ctx->targets_per_rank; + op_layout_test_mem += + sizeof(struct pool_component) * new_comps; + op_daos_api_mem += pool_buf_size(new_comps); + } + if (rc != 0) + break; + } + + /* + * Build the complete target list for this operation. + * All component-type logic (TARGET / RANK / NODE) is + * handled inside fetch_targets. + */ + rc = fetch_targets(ctx, &operations[i], &tgts); + if (rc != 0) { + D_ERROR("fetch_targets failed rc=%d\n", rc); + break; + } + + /* + * Apply the two-stage status change and collect the + * per-OID shard rebuild diff between the two stages. + */ + printf(" Applying status change and computing diff...\n"); + rc = set_tgt_status_and_find_diff(ctx, &tgts, operations[i].type, diff, + &ms_rebuild); + if (rc != 0) { + D_ERROR("set_tgt_status_and_find_diff failed rc=%d\n", rc); + break; + } + + /* Capture the post-operation layout */ + printf(" Capturing post-operation layouts...\n"); + rc = capture_layouts(ctx, post_layout, &ms_place2); + if (rc != 0) { + D_ERROR("capture_layouts failed rc=%d\n", rc); + break; + } + /* Count actual DAOS API allocation: struct header + ol_shards array */ + { + int k; + + for (k = 0; k < ctx->num_oids; k++) { + if (post_layout[k] != NULL) + op_daos_api_mem += + sizeof(*post_layout[k]) + + post_layout[k]->ol_nr * + sizeof(*post_layout[k]->ol_shards); + } + } + + /* + * Merge diff into orig_layout, verify it matches + * post_layout, and print the moved-shard percentage. + */ + printf(" Comparing layouts...\n"); + rc = compare_layout(ctx, orig_layout, diff, post_layout, max_diff); + if (rc != 0) { + fprintf(stderr, " Layout comparison failed rc=%d\n", rc); + break; + } + + printf(" Performance (pl_obj_place / pl_obj_find_rebuild only):\n"); + print_time_line("pl_obj_place (initial)", ms_place1); + print_time_line("pl_obj_find_rebuild", ms_rebuild); + print_time_line("pl_obj_place (post-op)", ms_place2); + print_time_line("Total", ms_place1 + ms_rebuild + ms_place2); + + printf(" Memory (this operation):\n"); + print_memory_line("Layout test memory", op_layout_test_mem); + print_memory_line("DAOS API memory", op_daos_api_mem); + print_memory_line("Total", op_layout_test_mem + op_daos_api_mem); + } while (0); + + pool_target_id_list_free(&tgts); + free_layouts(orig_layout, ctx->num_oids); + free_layouts(post_layout, ctx->num_oids); + free_diffs(diff, ctx->num_oids); + + if (rc != 0) + return rc; + } + + return 0; +} + +static int +parse_args(int argc, char **argv, int64_t *nodes, int64_t *ranks, int64_t *targets, + int64_t *obj_count, char **object_class_str, struct operation operations[], + int *operation_count) +{ + bool seen_n = false; + bool seen_r = false; + bool seen_t = false; + bool seen_c = false; + bool seen_N = false; + bool seen_o = false; + int rc; + + while (1) { + static struct option long_options[] = {{"nodes", required_argument, 0, 'n'}, + {"ranks", required_argument, 0, 'r'}, + {"targets", required_argument, 0, 't'}, + {"class", required_argument, 0, 'c'}, + {"obj-count", required_argument, 0, 'N'}, + {"operation", required_argument, 0, 'o'}, + {"help", no_argument, 0, 'h'}, + {0, 0, 0, 0}}; + + int opt; + + opt = getopt_long(argc, argv, "n:r:t:c:N:o:h", long_options, NULL); + if (opt == -1) + break; + +#define CHECK_DUPLICATE(flag, name) \ + do { \ + if (flag) { \ + fprintf(stderr, "ERROR: " name " may only be specified once\n"); \ + print_usage(argv[0]); \ + return -DER_INVAL; \ + } \ + flag = true; \ + } while (0) + + switch (opt) { + case 'n': + CHECK_DUPLICATE(seen_n, "-n/--nodes"); + rc = parse_int_arg("nodes", optarg, nodes); + if (rc != 0) + return rc; + break; + + case 'r': + CHECK_DUPLICATE(seen_r, "-r/--ranks"); + rc = parse_int_arg("ranks", optarg, ranks); + if (rc != 0) + return rc; + break; + + case 't': + CHECK_DUPLICATE(seen_t, "-t/--targets"); + rc = parse_int_arg("targets", optarg, targets); + if (rc != 0) + return rc; + break; + + case 'c': + CHECK_DUPLICATE(seen_c, "-c/--class"); + *object_class_str = optarg; + break; + + case 'N': + CHECK_DUPLICATE(seen_N, "-N/--obj-count"); + rc = parse_int_arg("obj-count", optarg, obj_count); + if (rc != 0) + return rc; + break; + + case 'o': { + CHECK_DUPLICATE(seen_o, "-o/--operation"); + char op_buf[1024]; + + strncpy(op_buf, optarg, sizeof(op_buf) - 1); + op_buf[sizeof(op_buf) - 1] = '\0'; + if (parse_operations(op_buf, operations, operation_count) != 0) { + fprintf(stderr, "ERROR: failed to parse --operation \"%s\"\n", + optarg); + return -DER_INVAL; + } + break; + } + + case 'h': + print_usage(argv[0]); + return 1; + default: + print_usage(argv[0]); + return -DER_INVAL; + } +#undef CHECK_DUPLICATE + } + + if (argc == 1) { + print_usage(argv[0]); + return -DER_INVAL; + } + + return 0; +} + +/* + * Initialize the test context from parsed arguments. + * + * Validates the configuration, sets up obj_class and pl subsystems, + * initializes pool map and placement map, allocates and generates the OID + * array, and records setup memory usage in ctx->setup_layout_test_mem and + * ctx->setup_daos_api_mem. + * + * @return 0 on success, negative DAOS error code on failure + */ +static int +config_setup(struct test_ctx *ctx, int64_t nodes, int64_t ranks, int64_t targets, int64_t obj_count, + char *object_class_str, struct operation *operations, int operation_count) +{ + int rc; + + memset(ctx, 0, sizeof(*ctx)); + rc = obj_class_init(); + if (rc != 0) { + D_ERROR("obj_class_init failed rc=%d\n", rc); + return rc; + } + pl_init(); + + rc = validate_configuration(nodes, ranks, targets, obj_count, object_class_str, operations, + operation_count); + if (rc != 0) { + obj_class_fini(); + pl_fini(); + return rc; + } + + printf("Running layout test with the following configuration:\n"); + printf("Nodes : %lld\n", (long long)nodes); + printf("Ranks per node : %lld\n", (long long)ranks); + printf("Targets per rank : %lld\n", (long long)targets); + printf("Object class : %s\n", object_class_str); + printf("Object count : %lld\n", (long long)obj_count); + + printf("\nSetting up layout...\n"); + ctx->nodes = (int)nodes; + ctx->ranks_per_node = (int)ranks; + ctx->targets_per_rank = (int)targets; + ctx->num_oids = (int)obj_count; + ctx->oclass = daos_oclass_name2id(object_class_str); + + uuid_generate(ctx->uuid); + pool_map_and_pl_map_init(ctx); + D_ASSERT(ctx->pool_map != NULL); + D_ASSERT(ctx->pl_map != NULL); + + D_ALLOC_ARRAY(ctx->oids, ctx->num_oids); + if (ctx->oids == NULL) { + obj_class_fini(); + cleanup(ctx); + return -DER_NOMEM; + } + ctx->setup_layout_test_mem += sizeof(*ctx->oids) * ctx->num_oids; + + printf("Setup memory (one-time, pool/pl-map init + OID array):\n"); + print_memory_line("Layout test memory", ctx->setup_layout_test_mem); + print_memory_line("DAOS API memory", ctx->setup_daos_api_mem); + print_memory_line("Total", ctx->setup_layout_test_mem + ctx->setup_daos_api_mem); + + generate_oids(ctx); + + return 0; +} + +int +main(int argc, char **argv) +{ + int64_t nodes = DEFAULT_NODES; + int64_t ranks = DEFAULT_RANKS; + int64_t targets = DEFAULT_TARGETS; + int64_t obj_count = DEFAULT_OBJ_COUNT; + char *object_class_str = DEFAULT_OBJ_CLASS; + struct operation operations[MAX_OPERATIONS]; + int operation_count = 0; + int rc; + struct test_ctx ctx; + + rc = parse_args(argc, argv, &nodes, &ranks, &targets, &obj_count, &object_class_str, + operations, &operation_count); + if (rc == 1) + return 0; + if (rc != 0) + return EXIT_FAILURE; + + rc = config_setup(&ctx, nodes, ranks, targets, obj_count, object_class_str, operations, + operation_count); + if (rc != 0) + return EXIT_FAILURE; + + /* + * Execute tests + */ + rc = execute_tests(&ctx, operations, operation_count); + if (rc != 0) { + printf("layout_test_runner failed rc=%d\n", rc); + D_FREE(ctx.oids); + obj_class_fini(); + cleanup(&ctx); + return EXIT_FAILURE; + } + printf("\nEnd of test...\n"); + + /* Cleanup and finalize */ + D_FREE(ctx.oids); + obj_class_fini(); + cleanup(&ctx); /* also calls pl_fini() */ + return 0; +} diff --git a/src/placement/tests/layout_test_helpers.c b/src/placement/tests/layout_test_helpers.c new file mode 100644 index 00000000000..cae98246c55 --- /dev/null +++ b/src/placement/tests/layout_test_helpers.c @@ -0,0 +1,797 @@ +/** + * (C) Copyright 2026 Hewlett Packard Enterprise Development LP + * + * SPDX-License-Identifier: BSD-2-Clause-Patent + */ + +#include + +#include "layout_test_helpers.h" +#include "../../pool/rpc.h" +#include "../../pool/srv_pool_map.h" + +void +pool_map_and_pl_map_init(struct test_ctx *ctx) +{ + struct pool_component *comps; + struct pool_component *comp; + struct pool_buf *buf; + int nr_nodes = ctx->nodes; + int nr_ranks = ctx->nodes * ctx->ranks_per_node; + int nr_tgts = ctx->nodes * ctx->ranks_per_node * ctx->targets_per_rank; + int nr = nr_nodes + nr_ranks + nr_tgts; + int i; + int rc; + + D_ALLOC_ARRAY(comps, nr); + D_ASSERT(comps != NULL); + ctx->setup_layout_test_mem += sizeof(*comps) * nr; + comp = comps; + + /* Nodes (fault domains) */ + for (i = 0; i < nr_nodes; i++, comp++) { + comp->co_type = PO_COMP_TP_NODE; + comp->co_status = PO_COMP_ST_UPIN; + comp->co_id = i; + comp->co_rank = i; + comp->co_ver = 1; + comp->co_nr = ctx->ranks_per_node; + } + + /* Ranks */ + for (i = 0; i < nr_ranks; i++, comp++) { + comp->co_type = PO_COMP_TP_RANK; + comp->co_status = PO_COMP_ST_UPIN; + comp->co_id = i; + comp->co_rank = i; + comp->co_ver = 1; + comp->co_nr = ctx->targets_per_rank; + } + + /* Targets */ + for (i = 0; i < nr_tgts; i++, comp++) { + comp->co_type = PO_COMP_TP_TARGET; + comp->co_status = PO_COMP_ST_UPIN; + comp->co_id = i; + comp->co_rank = i / ctx->targets_per_rank; + comp->co_index = i % ctx->targets_per_rank; + comp->co_ver = 1; + comp->co_nr = 1; + } + + buf = pool_buf_alloc(nr); + D_ASSERT(buf != NULL); + ctx->setup_daos_api_mem += pool_buf_size(nr); + rc = pool_buf_attach(buf, comps, nr); + assert_success(rc); + D_FREE(comps); + + rc = pool_map_create(buf, 1, &ctx->pool_map); + assert_success(rc); + D_FREE(buf); + + rc = pl_map_update(ctx->uuid, ctx->pool_map, true, PL_TYPE_JUMP_MAP); + assert_success(rc); + + ctx->pl_map = pl_map_find(ctx->uuid, DAOS_OBJ_NIL); + D_ASSERT(ctx->pl_map != NULL); + ctx->setup_daos_api_mem += sizeof(*ctx->pl_map); +} + +/* + * Rebuild the placement map under ctx->uuid from the current pool map, + * release the stale ctx->pl_map reference, and re-acquire the fresh one + * from the hash table. Must be called after every pool map mutation so + * that ctx->pl_map always reflects the latest topology. + */ +static int +refresh_pl_map(struct test_ctx *ctx) +{ + int rc; + + rc = pl_map_update(ctx->uuid, ctx->pool_map, false, PL_TYPE_JUMP_MAP); + if (rc != 0) { + D_ERROR("pl_map_update failed rc=%d\n", rc); + return rc; + } + pl_map_decref(ctx->pl_map); + ctx->pl_map = pl_map_find(ctx->uuid, DAOS_OBJ_NIL); + D_ASSERT(ctx->pl_map != NULL); + return 0; +} + +void +generate_oids(struct test_ctx *ctx) +{ + int i; + char ename[MAX_OBJ_CLASS_NAME_LEN]; + daos_oclass_id2name(ctx->oclass, ename); + printf("Generating %d OIDs with class %s...\n", ctx->num_oids, ename); + for (i = 0; i < ctx->num_oids; i++) { + ctx->oids[i].oid.omd_id.hi = 0; + ctx->oids[i].oid.omd_id.lo = ((uint64_t)rand() << 32) | i; + ctx->oids[i].oid.omd_ver = pool_map_get_version(ctx->pool_map); + ctx->oids[i].oid.omd_pda = 1; + daos_obj_set_oid_by_class(&ctx->oids[i].oid.omd_id, 0, ctx->oclass, 0); + } +} + +int +capture_layouts(struct test_ctx *ctx, struct pl_obj_layout **layouts, double *ms_place_out) +{ + int rc; + int i; + + *ms_place_out = 0.0; + for (i = 0; i < ctx->num_oids; i++) { + struct timespec _t0, _t1; + + clock_gettime(CLOCK_MONOTONIC, &_t0); + rc = pl_obj_place(ctx->pl_map, PLT_LAYOUT_VERSION, &ctx->oids[i].oid, 0, NULL, + &layouts[i]); + clock_gettime(CLOCK_MONOTONIC, &_t1); + *ms_place_out += + (_t1.tv_sec - _t0.tv_sec) * 1e3 + (_t1.tv_nsec - _t0.tv_nsec) / 1e6; + if (rc != 0 || layouts[i] == NULL) { + printf("placement failed for oid %d rc=%d\n", i, rc); + continue; + } + } + return 0; +} + +void +free_layouts(struct pl_obj_layout **layouts, int num_oids) +{ + int i; + + if (layouts == NULL) + return; + for (i = 0; i < num_oids; i++) { + if (layouts[i] != NULL) + pl_obj_layout_free(layouts[i]); + } + D_FREE(layouts); +} + +void +free_diffs(struct shard_diff *diff, int num_oids) +{ + int i; + + if (diff == NULL) + return; + for (i = 0; i < num_oids; i++) { + D_FREE(diff[i].shard_ids); + D_FREE(diff[i].spare_tgts); + } + D_FREE(diff); +} + +/** + * Perform two layout comparisons after an operation: + * + * Check 1 – Initial vs final (bounded drift): + * Compare orig_layout against post_layout directly. The number of shards + * whose target differs within each group must not exceed max_diff. This + * verifies that the operation did not displace more shards per group than + * the fault-tolerance budget allows. + * + * Check 2 – Diff applied to original must equal final exactly: + * Apply every rebuild diff entry (shard_ids / spare_tgts) to a temporary + * copy of orig_layout. The resulting merged layout is then compared + * against post_layout and must match target-for-target with zero + * mismatches. This verifies that the diff fully and precisely describes + * all shard movements. + * + * @param ctx Test context (num_oids, oclass) + * @param orig_layout Layout captured before the operation (not modified) + * @param diff Per-OID rebuild diff from set_tgt_status_and_find_diff() + * @param post_layout Layout captured after the final status change + * @param max_diff Maximum allowed shard target differences per group + * (Check 1 only; Check 2 requires an exact match) + */ +int +compare_layout(struct test_ctx *ctx, struct pl_obj_layout **orig_layout, struct shard_diff *diff, + struct pl_obj_layout **post_layout, int max_diff) +{ + int i; + int j; + int k; + int total_shards = 0; + int moved_shards = 0; + + /* ------------------------------------------------------------------ * + * Check 1: Compare initial vs final layout. * + * Per-group shard diff count must not exceed max_diff. * + * ------------------------------------------------------------------ */ + for (i = 0; i < ctx->num_oids; i++) { + for (j = 0; j < (int)orig_layout[i]->ol_grp_nr; j++) { + int shard_diff_cnt = 0; + + for (k = 0; k < (int)orig_layout[i]->ol_grp_size; k++) { + struct pl_obj_shard *s1; + struct pl_obj_shard *s2; + + s1 = + &orig_layout[i]->ol_shards[j * orig_layout[i]->ol_grp_size + k]; + s2 = + &post_layout[i]->ol_shards[j * post_layout[i]->ol_grp_size + k]; + + if (s1->po_target != s2->po_target) + shard_diff_cnt++; + } + + if (shard_diff_cnt > max_diff) { + fprintf(stderr, + " ERROR: OID %d grp %d: %d/%d shards differ " + "(max allowed: %d)\n", + i, j, shard_diff_cnt, orig_layout[i]->ol_grp_size, + max_diff); + return -DER_IO; + } + } + } + + /* ------------------------------------------------------------------ * + * Check 2: Apply diff to original, result must equal final exactly. * + * ------------------------------------------------------------------ */ + for (i = 0; i < ctx->num_oids; i++) { + total_shards += orig_layout[i]->ol_nr; + moved_shards += diff[i].nr; + } + + printf(" Moved: %d/%d shards (%.1f%%)\n", moved_shards, total_shards, + total_shards > 0 ? 100.0 * moved_shards / total_shards : 0.0); + + for (i = 0; i < ctx->num_oids; i++) { + struct pl_obj_shard *merged; + int nr = orig_layout[i]->ol_nr; + + /* Temporary copy of orig so we do not mutate the original. */ + D_ALLOC_ARRAY(merged, nr); + if (merged == NULL) { + fprintf(stderr, " ERROR: failed to allocate merged layout\n"); + return -DER_NOMEM; + } + memcpy(merged, orig_layout[i]->ol_shards, nr * sizeof(struct pl_obj_shard)); + + /* Apply diff entries to the copy. */ + for (j = 0; j < diff[i].nr; j++) { + uint32_t shard_idx = diff[i].shard_ids[j]; + uint32_t spare_tgt = diff[i].spare_tgts[j]; + + if (shard_idx < (uint32_t)nr) + merged[shard_idx].po_target = spare_tgt; + } + + /* Merged layout must match post_layout with zero mismatches. */ + for (j = 0; j < (int)orig_layout[i]->ol_grp_nr; j++) { + for (k = 0; k < (int)orig_layout[i]->ol_grp_size; k++) { + int idx = j * orig_layout[i]->ol_grp_size + k; + struct pl_obj_shard *s1 = &merged[idx]; + struct pl_obj_shard *s2 = &post_layout[i]->ol_shards[idx]; + + if (s1->po_target != s2->po_target) { + fprintf(stderr, + " ERROR: OID %d grp %d shard %d: " + "merged target %u != post target %u\n", + i, j, k, s1->po_target, s2->po_target); + D_FREE(merged); + return -DER_IO; + } + } + } + + D_FREE(merged); + } + printf(" Validation succeeded\n"); + return 0; +} + +int +validate_configuration(int64_t nodes, int64_t ranks, int64_t targets, int64_t obj_count, + const char *object_class_str, struct operation *operations, + int operation_count) +{ + struct daos_oclass_attr *oc; + int64_t total_ranks; + int64_t total_targets; + int64_t required_nodes; + int64_t required_targets; + int64_t grp; + uint32_t nr_grps = 0; + int cid; + int i; + int j; + + if (nodes <= 0 || ranks <= 0 || targets <= 0 || obj_count <= 0) { + fprintf( + stderr, + "ERROR: nodes, ranks, targets, and object count must be positive integers\n"); + return -DER_INVAL; + } + + total_ranks = nodes * ranks; + total_targets = total_ranks * targets; + + cid = daos_oclass_name2id(object_class_str); + if (cid == DAOS_OC_UNKNOWN) { + fprintf(stderr, "ERROR: unknown object class: %s\n", object_class_str); + return -DER_INVAL; + } + + oc = daos_oclass_id2attr(cid, &nr_grps); + if (!oc) { + D_DEBUG(DB_PL, "Unknown object class %u\n", (unsigned int)cid); + return -DER_INVAL; + } + + switch (oc->ca_resil) { + case DAOS_RES_REPL: + required_nodes = oc->u.rp.r_num; + grp = oc->ca_grp_nr; + break; + case DAOS_RES_EC: + required_nodes = oc->u.ec.e_k + oc->u.ec.e_p; + grp = oc->ca_grp_nr; + break; + default: + fprintf(stderr, "ERROR: unsupported oclass resilience type\n"); + return -DER_INVAL; + } + + if (nodes < required_nodes) { + fprintf(stderr, + "ERROR: The given object class requires minimum %lld number of " + "fault domains/nodes\n", + (long long)required_nodes); + return -DER_INVAL; + } + + /* + * For GX classes ca_grp_nr == DAOS_OBJ_GRP_MAX (65535); the actual + * group count is resolved at placement time from the available targets. + * The minimum viable pool only needs one group, i.e. required_nodes + * targets. Use that lower bound instead of the sentinel value. + */ + required_targets = required_nodes * (grp == DAOS_OBJ_GRP_MAX ? 1 : grp); + if (total_targets < required_targets) { + fprintf(stderr, + "ERROR: insufficient targets available: required %lld, provided %lld\n", + (long long)required_targets, (long long)total_targets); + return -DER_INVAL; + } + + if (operation_count == 0) { + fprintf(stderr, "ERROR: no operations specified (use --operation)\n"); + return -DER_INVAL; + } + + for (i = 0; i < operation_count; i++) { + struct operation *op = &operations[i]; + + if (op->nr_args == 0) { + fprintf(stderr, "ERROR: operation %d has no arguments\n", i); + return -DER_INVAL; + } + + /* + * For OP_ADD, node IDs must be assigned sequentially: the first + * arg must equal the current node count, the second arg must be + * current + 1, and so on. Gaps or duplicates are rejected. + * After validation, update the running topology counts so that + * subsequent operations in the list can reference the new nodes, + * ranks, and targets. + */ + if (op->type == OP_ADD) { + for (j = 0; j < op->nr_args; j++) { + if ((int64_t)op->args[j] != nodes + j) { + fprintf(stderr, + "ERROR: op %d: node ID %d is not the next " + "sequential node (expected %lld)\n", + i, op->args[j], (long long)(nodes + j)); + return -DER_INVAL; + } + } + nodes += op->nr_args; + total_ranks += (int64_t)op->nr_args * ranks; + total_targets += (int64_t)op->nr_args * ranks * targets; + continue; + } + + /* Validate each argument ID is within range for the given mode */ + for (j = 0; j < op->nr_args; j++) { + int id = op->args[j]; + + if (id < 0) { + fprintf(stderr, "ERROR: operation %d: negative ID %d\n", i, id); + return -DER_INVAL; + } + + switch (op->component) { + case RANK: + if ((int64_t)id >= total_ranks) { + fprintf(stderr, + "ERROR: op %d: rank %d out of range (total: %ld)\n", + i, id, total_ranks); + return -DER_INVAL; + } + break; + case NODE: + if ((int64_t)id >= nodes) { + fprintf(stderr, + "ERROR: op %d: node %d out of range (total: %ld)\n", + i, id, nodes); + return -DER_INVAL; + } + break; + case TARGET: + if ((int64_t)id >= total_targets) { + fprintf( + stderr, + "ERROR: op %d: target %d out of range (total: %ld)\n", + i, id, total_targets); + return -DER_INVAL; + } + break; + default: + fprintf(stderr, "ERROR: op %d: invalid mode %d\n", i, + op->component); + return -DER_INVAL; + } + } + } + + return 0; +} + +void +cleanup(struct test_ctx *ctx) +{ + if (ctx->pl_map != NULL) { + pl_map_decref(ctx->pl_map); + ctx->pl_map = NULL; + } + if (ctx->pool_map != NULL) + pool_map_decref(ctx->pool_map); + + pl_fini(); +} + +/** + * Build a flat list of all target IDs for a given operation. + * + * Handles all three component types: + * TARGET – args are raw target IDs; copied directly into the output list. + * RANK – each arg is a rank ID; all targets of that rank are merged in. + * NODE – each arg is a node ID; all targets of that node are merged in. + * + * @param ctx Test context + * @param op Operation describing component type and ID arguments + * @param tgts Output target ID list (caller must free with pool_target_id_list_free) + * + * @return 0 on success, negative error on failure + */ +int +fetch_targets(struct test_ctx *ctx, struct operation *op, struct pool_target_id_list *tgts) +{ + int a; + int rc = 0; + + if (op->component == TARGET) { + rc = pool_target_id_list_alloc(op->nr_args, tgts); + if (rc != 0) + return rc; + for (a = 0; a < op->nr_args; a++) + tgts->pti_ids[a].pti_id = op->args[a]; + } else { + pool_comp_type_t comp_type = + (op->component == RANK) ? PO_COMP_TP_RANK : PO_COMP_TP_NODE; + + for (a = 0; a < op->nr_args; a++) { + struct pool_target_id_list tmp = {0}; + struct pool_domain *domain = NULL; + int t; + + rc = pool_map_find_domain(ctx->pool_map, comp_type, op->args[a], &domain); + if (rc != 1) { + printf("ERROR: Cannot find domain type=%d id=%d (rc=%d)\n", + comp_type, op->args[a], rc); + return -DER_NONEXIST; + } + + D_ALLOC_ARRAY(tmp.pti_ids, domain->do_target_nr); + if (tmp.pti_ids == NULL) + return -DER_NOMEM; + + tmp.pti_number = domain->do_target_nr; + for (t = 0; t < domain->do_target_nr; t++) + tmp.pti_ids[t].pti_id = domain->do_targets[t].ta_comp.co_id; + + rc = pool_target_id_list_merge(tgts, &tmp); + pool_target_id_list_free(&tmp); + if (rc != 0) + return rc; + } + } + + return 0; +} + +/** + * Apply a two-stage component status change for the given target list and + * collect the rebuild diff between the two stages. + * + * Stage 1 sets the intermediate status (e.g. DOWN for exclude), then calls + * pl_obj_find_rebuild() for every OID to discover which shards need to move + * and where their spare targets are. Stage 2 advances to the final status + * (e.g. DOWNOUT for exclude) and refreshes the placement map. + * + * Two-stage mapping: + * OP_EXCLUDE : DOWN -> DOWNOUT + * OP_DRAIN : DRAIN -> DOWNOUT + * OP_REINT : UP -> UPIN + * OP_ADD : (NEW already set by add_node) -> UPIN + * + * @param ctx Test context (pool_map, pl_map, uuid, num_oids) + * @param tgts List of target IDs to update + * @param op_type Operation type (OP_EXCLUDE, OP_REINT, OP_DRAIN, OP_ADD) + * @param diff Output array (length ctx->num_oids); each entry receives + * the shard indices and spare target IDs for that OID + * @param ms_rebuild_out Accumulated wall-clock time (ms) spent in pl_obj_find_rebuild() + */ +int +set_tgt_status_and_find_diff(struct test_ctx *ctx, struct pool_target_id_list *tgts, + enum operation_type op_type, struct shard_diff *diff, + double *ms_rebuild_out) +{ + struct daos_obj_md md = {0}; + struct timespec t0, t1; + uint32_t *spare_tgts = NULL; + uint32_t *shard_ids = NULL; + uint32_t buf_size = MAX_SHARDS; + uint32_t ver; + int opc1; + int opc2; + int rc; + int i; + + /* Map operation type to the two sequential map opcodes */ + switch (op_type) { + case OP_EXCLUDE: + opc1 = MAP_EXCLUDE; + opc2 = MAP_EXCLUDE_OUT; + break; + case OP_DRAIN: + opc1 = MAP_DRAIN; + opc2 = MAP_EXCLUDE_OUT; + break; + case OP_REINT: + opc1 = MAP_REINT; + opc2 = MAP_ADD_IN; + break; + case OP_ADD: + /* + * update_one_dom is called by ds_pool_map_tgts_update() only when + * rc > 0 (i.e., the target status actually changed) or exclude_rank == true. + * For MAP_ADD_IN, targets already-UPIN return rc=0 (nothing to do), + * so update_one_dom is skipped for those — but for targets transitioning UP→UPIN + * it returns rc=1 and update_one_dom is reached. + * Since MAP_ADD_IN has no case in update_one_dom (falls through to default: break), + * the domain is never promoted to UPIN. So, use MAP_EXTEND + MAP_FINISH_REBUILD + * instead, which does promote the domain to UPIN. + */ + opc1 = MAP_EXTEND; + opc2 = MAP_FINISH_REBUILD; + break; + default: + D_ERROR("Invalid operation type %d\n", op_type); + return -DER_INVAL; + } + + /* Stage 1: advance targets to the intermediate status */ + rc = ds_pool_map_tgts_update(NULL, ctx->pool_map, tgts, opc1, false, NULL, false); + if (rc != 0) { + D_ERROR("ds_pool_map_tgts_update (stage 1) failed rc=%d\n", rc); + return rc; + } + pool_map_update_failed_cnt(ctx->pool_map); + pool_map_init_in_fseq(ctx->pool_map); + ver = pool_map_get_version(ctx->pool_map); + rc = refresh_pl_map(ctx); + if (rc != 0) + return rc; + + /* + * Identify which shards need to be rebuilt/moved and their + * spare targets at the intermediate state. Scratch buffers start + * at MAX_SHARDS entries; if pl_obj_find_rebuild returns -DER_REC2BIG + * they are grown by 64 and the call is retried. The grown buffers + * are reused across OIDs to avoid repeated allocation. + */ + D_ALLOC_ARRAY(spare_tgts, buf_size); + if (spare_tgts == NULL) + return -DER_NOMEM; + D_ALLOC_ARRAY(shard_ids, buf_size); + if (shard_ids == NULL) { + D_FREE(spare_tgts); + return -DER_NOMEM; + } + + *ms_rebuild_out = 0.0; + for (i = 0; i < ctx->num_oids; i++) { + int j; + + diff[i].nr = 0; + diff[i].shard_ids = NULL; + diff[i].spare_tgts = NULL; + md = ctx->oids[i].oid; + md.omd_pda = 1; + md.omd_ver = ver; + +retry: + clock_gettime(CLOCK_MONOTONIC, &t0); + diff[i].nr = pl_obj_find_rebuild(ctx->pl_map, PLT_LAYOUT_VERSION, &md, NULL, ver, + spare_tgts, shard_ids, buf_size); + clock_gettime(CLOCK_MONOTONIC, &t1); + *ms_rebuild_out += (t1.tv_sec - t0.tv_sec) * 1e3 + (t1.tv_nsec - t0.tv_nsec) / 1e6; + + if (diff[i].nr == -DER_REC2BIG) { + /* + * Output buffer too small; grow both arrays by 64 and retry. + */ + uint32_t new_size = buf_size + 64; + uint32_t *new_spare; + uint32_t *new_shard; + + D_ALLOC_ARRAY(new_spare, new_size); + D_ALLOC_ARRAY(new_shard, new_size); + if (new_spare == NULL || new_shard == NULL) { + D_FREE(new_spare); + D_FREE(new_shard); + D_FREE(spare_tgts); + D_FREE(shard_ids); + return -DER_NOMEM; + } + D_FREE(spare_tgts); + D_FREE(shard_ids); + spare_tgts = new_spare; + shard_ids = new_shard; + buf_size = new_size; + goto retry; + } + + if (diff[i].nr < 0) + diff[i].nr = 0; + + if (diff[i].nr > 0) { + D_ALLOC_ARRAY(diff[i].shard_ids, diff[i].nr); + if (diff[i].shard_ids == NULL) { + D_FREE(spare_tgts); + D_FREE(shard_ids); + return -DER_NOMEM; + } + D_ALLOC_ARRAY(diff[i].spare_tgts, diff[i].nr); + if (diff[i].spare_tgts == NULL) { + D_FREE(diff[i].shard_ids); + D_FREE(spare_tgts); + D_FREE(shard_ids); + return -DER_NOMEM; + } + for (j = 0; j < diff[i].nr; j++) { + diff[i].shard_ids[j] = shard_ids[j]; + diff[i].spare_tgts[j] = spare_tgts[j]; + } + } + } + D_FREE(spare_tgts); + D_FREE(shard_ids); + + /* Stage 2: advance targets to the final status */ + rc = ds_pool_map_tgts_update(NULL, ctx->pool_map, tgts, opc2, false, NULL, false); + if (rc != 0) { + D_ERROR("ds_pool_map_tgts_update (stage 2) failed rc=%d\n", rc); + return rc; + } + pool_map_update_failed_cnt(ctx->pool_map); + rc = refresh_pl_map(ctx); + if (rc != 0) + return rc; + return 0; +} + +/** + * Add a new node to the pool map. + * + * Allocates one node, ranks_per_node ranks, and + * ranks_per_node * targets_per_rank targets, all with PO_COMP_ST_NEW status. + * IDs are assigned sequentially after the current pool map population. + * The pool version is bumped. + * + * @param ctx Test context (pool_map, ranks_per_node, targets_per_rank) + * @param node_id ID to assign to the new node + * + * @return 0 on success, negative DAOS error code on failure + */ +int +add_node(struct test_ctx *ctx, uint32_t node_id) +{ + struct pool_component *comps; + struct pool_component *comp; + struct pool_buf *buf; + uint32_t ver; + uint32_t base_rank; + uint32_t base_tgt; + int nr_ranks; + int nr_tgts; + int nr_comps; + int r; + int t; + int rc; + + nr_ranks = ctx->ranks_per_node; + nr_tgts = ctx->ranks_per_node * ctx->targets_per_rank; + nr_comps = 1 + nr_ranks + nr_tgts; + + /* New IDs continue from the current pool map population */ + base_rank = node_id * ctx->ranks_per_node; + base_tgt = pool_map_find_target(ctx->pool_map, PO_COMP_ID_ALL, NULL); + ver = pool_map_get_version(ctx->pool_map) + 1; + + D_ALLOC_ARRAY(comps, nr_comps); + if (comps == NULL) + return -DER_NOMEM; + + comp = comps; + + /* Node */ + comp->co_type = PO_COMP_TP_NODE; + comp->co_status = PO_COMP_ST_NEW; + comp->co_id = node_id; + comp->co_rank = 0; + comp->co_ver = ver; + comp->co_fseq = 1; + comp->co_nr = nr_ranks; + comp++; + + /* Ranks (one per ranks_per_node slot) */ + for (r = 0; r < nr_ranks; r++, comp++) { + comp->co_type = PO_COMP_TP_RANK; + comp->co_status = PO_COMP_ST_NEW; + comp->co_id = base_rank + r; + comp->co_rank = base_rank + r; + comp->co_ver = ver; + comp->co_fseq = 1; + comp->co_nr = ctx->targets_per_rank; + } + + /* Targets */ + for (t = 0; t < nr_tgts; t++, comp++) { + comp->co_type = PO_COMP_TP_TARGET; + comp->co_status = PO_COMP_ST_NEW; + comp->co_id = base_tgt + t; + comp->co_rank = base_rank + t / ctx->targets_per_rank; + comp->co_index = t % ctx->targets_per_rank; + comp->co_ver = ver; + comp->co_fseq = 1; + comp->co_nr = 1; + } + + buf = pool_buf_alloc(nr_comps); + if (buf == NULL) { + D_FREE(comps); + return -DER_NOMEM; + } + + rc = pool_buf_attach(buf, comps, nr_comps); + D_FREE(comps); + if (rc != 0) { + D_FREE(buf); + return rc; + } + + rc = pool_map_extend(ctx->pool_map, ver, buf); + D_FREE(buf); + return rc; +} \ No newline at end of file diff --git a/src/placement/tests/layout_test_helpers.h b/src/placement/tests/layout_test_helpers.h new file mode 100644 index 00000000000..763aadbacf2 --- /dev/null +++ b/src/placement/tests/layout_test_helpers.h @@ -0,0 +1,102 @@ +/** + * (C) Copyright 2026 Hewlett Packard Enterprise Development LP + * + * SPDX-License-Identifier: BSD-2-Clause-Patent + */ +#include + +#include +#include +#include "place_obj_common.h" +#include +#include + +#define DEFAULT_NODES 8 +#define DEFAULT_RANKS 2 +#define DEFAULT_TARGETS 16 +#define DEFAULT_OBJ_COUNT 100000 +#define DEFAULT_OBJ_CLASS "EC_2P1G1" + +#define MAX_OPERATIONS 32 +#define MAX_OP_ARGS 32 +#define MAX_SHARDS 128 + +struct test_ctx { + int nodes; + int ranks_per_node; + int targets_per_rank; + int num_oids; + uint64_t setup_layout_test_mem; + uint64_t setup_daos_api_mem; + struct test_oid *oids; + uuid_t uuid; + daos_oclass_id_t oclass; + struct pool_map *pool_map; + struct pl_map *pl_map; +}; + +struct test_oid { + struct daos_obj_md oid; +}; + +struct oid_layout { + daos_obj_id_t oid; + uint32_t nr; + uint32_t grp_size; + uint32_t grp_nr; + struct pl_obj_shard *ol_shards; +}; + +enum operation_type { OP_EXCLUDE, OP_REINT, OP_DRAIN, OP_ADD, OP_INVALID }; + +enum operation_component { RANK, NODE, TARGET, INVALID }; + +struct operation { + enum operation_type type; + enum operation_component component; + int args[MAX_OP_ARGS]; + int nr_args; +}; + +struct shard_diff { + uint32_t *shard_ids; + uint32_t *spare_tgts; + int nr; +}; + +void +pool_map_and_pl_map_init(struct test_ctx *ctx); + +void +generate_oids(struct test_ctx *ctx); + +int +capture_layouts(struct test_ctx *ctx, struct pl_obj_layout **layouts, double *ms_place_out); + +void +free_layouts(struct pl_obj_layout **layouts, int num_oids); + +void +free_diffs(struct shard_diff *diff, int num_oids); + +int +compare_layout(struct test_ctx *ctx, struct pl_obj_layout **orig_layout, struct shard_diff *diff, + struct pl_obj_layout **post_layout, int max_diff); + +int +validate_configuration(int64_t nodes, int64_t ranks, int64_t targets, int64_t obj_count, + const char *object_class_str, struct operation *operations, + int operation_count); + +void +cleanup(struct test_ctx *ctx); + +int +fetch_targets(struct test_ctx *ctx, struct operation *op, struct pool_target_id_list *tgts); +int +set_tgt_status_and_find_diff(struct test_ctx *ctx, struct pool_target_id_list *tgts, + enum operation_type op_type, struct shard_diff *diff, + double *ms_rebuild_out); + +int +add_node(struct test_ctx *ctx, uint32_t node_id); \ No newline at end of file diff --git a/src/tests/ftest/SConscript b/src/tests/ftest/SConscript index 8b40034ffc0..4cff9e452d9 100644 --- a/src/tests/ftest/SConscript +++ b/src/tests/ftest/SConscript @@ -18,7 +18,7 @@ def scons(): 'daos_perf', 'daos_racer', 'daos_vol', 'daos_test', 'data', 'fault_domain', 'io', 'ior', 'mdtest', 'network', 'nvme', 'mpiio', - 'object', 'osa', 'pool', 'rebuild', 'recovery', 'security', + 'object', 'osa', 'placement', 'pool', 'rebuild', 'recovery', 'security', 'server', 'soak', 'erasurecode', 'datamover', 'scripts', 'dbench', 'harness', 'telemetry', 'deployment', 'performance', diff --git a/src/tests/ftest/placement/layout_test.py b/src/tests/ftest/placement/layout_test.py new file mode 100644 index 00000000000..c3b24d85e5d --- /dev/null +++ b/src/tests/ftest/placement/layout_test.py @@ -0,0 +1,236 @@ +""" +(C) Copyright 2026 Hewlett Packard Enterprise Development LP + +SPDX-License-Identifier: BSD-2-Clause-Patent +""" + +import os +import re +import shutil +import subprocess # nosec + +from avocado import Test + +# Resolve the repo root at import time, while CWD is still the source tree. +# This file lives at /src/tests/ftest/placement/layout_test.py +# so four levels up is the repo root. +_THIS_DIR = os.path.dirname(os.path.abspath(__file__)) +_REPO_ROOT = os.path.abspath(os.path.join(_THIS_DIR, "..", "..", "..", "..")) + + +class LayoutTest(Test): + """Placement layout computation performance test. + + Standalone test using plain avocado.Test — no DAOS server/agent + infrastructure required. Runs the layout_test binary directly and + parses per-phase timing output. + + Binary search order: + 1. DAOS_TEST_PREFIX environment variable (e.g. /path/to/install) + 2. Sibling install/ directory relative to the repo root + 3. PATH + + :avocado: recursive + """ + + # ------------------------------------------------------------------ + # helpers + # ------------------------------------------------------------------ + + def _find_binary(self): + """Locate the layout_test binary. + + Returns: + str: absolute path to layout_test + """ + candidates = [] + + prefix = os.environ.get("DAOS_TEST_PREFIX") + if prefix: + candidates.append(os.path.join(prefix, "bin", "layout_test")) + + # repo-relative: resolved at import time before avocado changes CWD + candidates.append(os.path.join(_REPO_ROOT, "install", "bin", "layout_test")) + + path_binary = shutil.which("layout_test") + if path_binary: + candidates.append(path_binary) + + for path in candidates: + if os.path.isfile(path) and os.access(path, os.X_OK): + return path + + self.cancel( + "layout_test binary not found. " + "Set DAOS_TEST_PREFIX or add install/bin to PATH." + ) + return None + + # pylint: disable=too-many-arguments, too-many-positional-arguments + def _run_binary( + self, binary, nodes, ranks, targets, object_class, obj_count, operation + ): + """Run layout_test and return stdout. + + Args: + binary (str): path to layout_test + nodes (int): number of nodes + ranks (int): ranks per node + targets (int): targets per rank + object_class (str): DAOS object class name + obj_count (int): number of OIDs + operation (str): operation sequence string + + Returns: + str: combined stdout from the process + """ + cmd = [ + binary, + "--nodes", + str(nodes), + "--ranks", + str(ranks), + "--targets", + str(targets), + "--class", + object_class, + "--obj-count", + str(obj_count), + "--operation", + operation, + ] + + self.log.info("Running: %s", " ".join(cmd)) + + try: + proc = subprocess.run( # nosec + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + timeout=self.timeout, + check=False, + encoding="utf-8", + ) + except subprocess.TimeoutExpired: + self.fail(f"layout_test timed out after {self.timeout}s") + + output = proc.stdout or "" + self.log.info(output) + + if proc.returncode != 0: + self.fail( + f"layout_test exited with code {proc.returncode}.\n" + f"Output:\n{output}" + ) + + return output + + # pylint: disable=too-many-locals + def _report_performance( + self, output, nodes, ranks, targets, object_class, obj_count, operation + ): + """Parse layout_test stdout and log structured performance data. + + Lines of interest emitted by layout_test for each operation:: + + Performance (pl_obj_place / pl_obj_find_rebuild only): + pl_obj_place (initial) : 45.123 ms + pl_obj_find_rebuild : 120.456 ms + pl_obj_place (post-op) : 44.789 ms + Total : 210.368 ms + + Args: + output (str): stdout text from layout_test + nodes (int): node count + ranks (int): ranks per node + targets (int): targets per rank + object_class (str): DAOS object class name + obj_count (int): number of OIDs exercised + operation (str): operation sequence string + """ + self.log.info("=" * 70) + self.log.info("LAYOUT PERFORMANCE SUMMARY") + self.log.info( + " Topology : %d nodes x %d ranks x %d targets", nodes, ranks, targets + ) + self.log.info(" Class : %-12s OIDs: %d", object_class, obj_count) + self.log.info(" Ops : %s", operation) + self.log.info("-" * 70) + + current_op = None + in_perf = False + found_any = False + + phase_pattern = re.compile( + r"^(pl_obj_place\s+\(initial\)|pl_obj_find_rebuild" + r"|pl_obj_place\s+\(post-op\)|Total)" + r"\s*:\s*([\d.]+)\s*ms$" + ) + + op_pattern = re.compile( + r"^\[(\d+)/(\d+)\]\s+Running\s+(\S+)\s+(\S+)\s+operation$" + ) + + for line in output.splitlines(): + stripped = line.strip() + + op_m = op_pattern.match(stripped) + if op_m: + current_op = f"{op_m.group(3)} {op_m.group(4)}" + in_perf = False + continue + + if "Performance (pl_obj_place / pl_obj_find_rebuild only)" in stripped: + in_perf = True + continue + + if in_perf: + m = phase_pattern.match(stripped) + if m: + label = m.group(1) + ms = float(m.group(2)) + self.log.info( + " [%-22s] %-30s: %10.3f ms", current_op or "?", label, ms + ) + found_any = True + elif stripped: + in_perf = False + + self.log.info("=" * 70) + + if not found_any: + self.fail( + "No performance data found in layout_test output. " + "Ensure the binary was built with timing support." + ) + + # ------------------------------------------------------------------ + # test method + # ------------------------------------------------------------------ + + def test_layout_performance(self): + """Run layout_test and report per-phase placement timing. + + :avocado: tags=all + :avocado: tags=vm + :avocado: tags=placement,layout + :avocado: tags=LayoutTest,test_layout_performance + """ + nodes = self.params.get("nodes", "/run/*", 32) + ranks = self.params.get("ranks", "/run/*", 1) + targets = self.params.get("targets", "/run/*", 8) + obj_count = self.params.get("obj_count", "/run/*", 100000) + object_class = self.params.get("object_class", "/run/object_class/*", "RP_3G1") + operation = self.params.get( + "operation", "/run/*", "exclude node=[0],reint node=[0]" + ) + + binary = self._find_binary() + + output = self._run_binary( + binary, nodes, ranks, targets, object_class, obj_count, operation + ) + + self._report_performance( + output, nodes, ranks, targets, object_class, obj_count, operation + ) diff --git a/src/tests/ftest/placement/layout_test.yaml b/src/tests/ftest/placement/layout_test.yaml new file mode 100644 index 00000000000..606968de0b2 --- /dev/null +++ b/src/tests/ftest/placement/layout_test.yaml @@ -0,0 +1,126 @@ +## +## (C) Copyright 2026 Hewlett Packard Enterprise Development LP +## +## SPDX-License-Identifier: BSD-2-Clause-Patent +## +## Comprehensive layout_test performance matrix covering all supported +## redundant object classes (21 variants). +## +## Fixed topology: 32 nodes x 1 rank x 8 targets (256 targets total). +## This satisfies the minimum-node requirement for every class below, +## including the most demanding one (EC_16P3: needs 19 nodes). +## +## Non-redundant classes (S1, S2, SX, etc.) are intentionally excluded +## because exclude→reint is meaningless when there is no spare target. +## +## Object class mux structure: +## +## Replication (5 factors, G1 and GX per factor = 10 variants): +## RP_2, RP_3, RP_4, RP_5, RP_6 +## +## Erasure Coding (11 configurations, G1 per config = 11 variants): +## 2P1, 2P2, 4P1, 4P2, 4P3, 8P1, 8P2, 8P3, 16P1, 16P2, 16P3 +## + +timeouts: + test_layout_performance: 300 + +# Fixed topology – adequate for all classes listed below +nodes: 32 +ranks: 1 +targets: 8 + +# Number of OIDs exercised per variant +obj_count: 100000 + +# Operation sequence applied to every variant +operation: "exclude node=[0],reint node=[0]" + +# One variant per supported object class +object_class: + !mux + + # ----------------------------------------------------------------------- + # Replication classes + # G1 = 1 placement group (minimum sharding) + # GX = max placement groups (stress the placement algorithm) + # ----------------------------------------------------------------------- + + rp2_g1: + object_class: RP_2G1 + + rp2_gx: + object_class: RP_2GX + + rp3_g1: + object_class: RP_3G1 + + rp3_gx: + object_class: RP_3GX + + rp4_g1: + object_class: RP_4G1 + + rp4_gx: + object_class: RP_4GX + + rp5_g1: + object_class: RP_5G1 + + rp5_gx: + object_class: RP_5GX + + rp6_g1: + object_class: RP_6G1 + + rp6_gx: + object_class: RP_6GX + + # ----------------------------------------------------------------------- + # Erasure Coding classes (k data + p parity, 1 placement group) + # Minimum nodes required = k + p + # ----------------------------------------------------------------------- + + # 2+1 = 3 nodes minimum + ec_2p1_g1: + object_class: EC_2P1G1 + + # 2+2 = 4 nodes minimum + ec_2p2_g1: + object_class: EC_2P2G1 + + # 4+1 = 5 nodes minimum + ec_4p1_g1: + object_class: EC_4P1G1 + + # 4+2 = 6 nodes minimum + ec_4p2_g1: + object_class: EC_4P2G1 + + # 4+3 = 7 nodes minimum + ec_4p3_g1: + object_class: EC_4P3G1 + + # 8+1 = 9 nodes minimum + ec_8p1_g1: + object_class: EC_8P1G1 + + # 8+2 = 10 nodes minimum + ec_8p2_g1: + object_class: EC_8P2G1 + + # 8+3 = 11 nodes minimum + ec_8p3_g1: + object_class: EC_8P3G1 + + # 16+1 = 17 nodes minimum + ec_16p1_g1: + object_class: EC_16P1G1 + + # 16+2 = 18 nodes minimum + ec_16p2_g1: + object_class: EC_16P2G1 + + # 16+3 = 19 nodes minimum + ec_16p3_g1: + object_class: EC_16P3G1