From 8c1a0a49e731be1eda7ac7e3849925985b029921 Mon Sep 17 00:00:00 2001 From: rasapala Date: Tue, 24 Feb 2026 13:56:17 +0100 Subject: [PATCH 01/33] Add lfs curl resume --- third_party/libgit2/lfs.patch | 41 +++++++++++++++++++++++++++++++++-- 1 file changed, 39 insertions(+), 2 deletions(-) diff --git a/third_party/libgit2/lfs.patch b/third_party/libgit2/lfs.patch index 6139df9e44..8e07425928 100644 --- a/third_party/libgit2/lfs.patch +++ b/third_party/libgit2/lfs.patch @@ -312,10 +312,10 @@ index 58cb4b424..00ddee9f3 100644 git_writestream **out, diff --git a/src/libgit2/lfs_filter.c b/src/libgit2/lfs_filter.c new file mode 100644 -index 000000000..484811a0c +index 000000000..a682cdbef --- /dev/null +++ b/src/libgit2/lfs_filter.c -@@ -0,0 +1,567 @@ +@@ -0,0 +1,604 @@ +/* +/ Copyright 2025 Intel Corporation +/ @@ -824,6 +824,43 @@ index 000000000..484811a0c + print_download_info(la->full_path, get_digit(la->lfs_size)); + /* Perform the request, res gets the return code */ + res = curl_easy_perform(dl_curl); ++ /* Check for resume of partial download error */ ++ if (res == CURLE_PARTIAL_FILE) { ++ curl_off_t resume_from = 0; ++ curl_easy_getinfo( ++ dl_curl, CURLINFO_CONTENT_LENGTH_DOWNLOAD_T, ++ &resume_from); ++ ++ if (resume_from == -1) { ++ fprintf(stderr, ++ "curl_easy_perform() failed with transferred a partial file error and server does not support range/resume.\n"); ++ } else { ++ fprintf(stderr, ++ "curl_easy_perform() failed with transferred a partial file error and trying to resume\n"); ++ curl_off_t offset = 0; ++ if (ftpfile.stream) { ++ fseek(ftpfile.stream, 0, SEEK_END); ++ offset = ftell(ftpfile.stream); ++ } else { ++ ftpfile.stream = ++ fopen(ftpfile.filename, "ab+"); ++ if (ftpfile.stream) { ++ fseek(ftpfile.stream, 0, ++ SEEK_END); ++ offset = ftell(ftpfile.stream); ++ } ++ } ++ ++ // Tell libcurl to resume ++ curl_easy_setopt( ++ dl_curl, CURLOPT_RESUME_FROM_LARGE, ++ offset); ++ /* Perform the request, res gets the return code ++ */ ++ res = curl_easy_perform(dl_curl); ++ } ++ } ++ + /* Check for errors */ + if (res != CURLE_OK) { + fprintf(stderr, "curl_easy_perform() failed: %s\n", From 43c8aac2b8ca1baf48849771134f1855ae5a10ae Mon Sep 17 00:00:00 2001 From: rasapala Date: Wed, 25 Feb 2026 12:05:53 +0100 Subject: [PATCH 02/33] Fix linux build --- third_party/libgit2/lfs.patch | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/third_party/libgit2/lfs.patch b/third_party/libgit2/lfs.patch index 8e07425928..9d09b48943 100644 --- a/third_party/libgit2/lfs.patch +++ b/third_party/libgit2/lfs.patch @@ -312,10 +312,10 @@ index 58cb4b424..00ddee9f3 100644 git_writestream **out, diff --git a/src/libgit2/lfs_filter.c b/src/libgit2/lfs_filter.c new file mode 100644 -index 000000000..a682cdbef +index 000000000..3b42a7ba4 --- /dev/null +++ b/src/libgit2/lfs_filter.c -@@ -0,0 +1,604 @@ +@@ -0,0 +1,603 @@ +/* +/ Copyright 2025 Intel Corporation +/ @@ -851,12 +851,11 @@ index 000000000..a682cdbef + } + } + -+ // Tell libcurl to resume ++ /* Tell libcurl to resume */ + curl_easy_setopt( + dl_curl, CURLOPT_RESUME_FROM_LARGE, + offset); -+ /* Perform the request, res gets the return code -+ */ ++ /* Perform the request, res gets the return code */ + res = curl_easy_perform(dl_curl); + } + } From f7e1c852fb709cd7aedf5e23dbbd60c7974734e0 Mon Sep 17 00:00:00 2001 From: Rafal Sapala Date: Fri, 27 Feb 2026 11:05:06 +0100 Subject: [PATCH 03/33] Git status --- src/pull_module/libgit2.cpp | 187 ++++++++++++++++++++++++++++++++++++ src/pull_module/libgit2.hpp | 2 + src/status.cpp | 3 + src/status.hpp | 2 + 4 files changed, 194 insertions(+) diff --git a/src/pull_module/libgit2.cpp b/src/pull_module/libgit2.cpp index 1566805c74..395d461a7f 100644 --- a/src/pull_module/libgit2.cpp +++ b/src/pull_module/libgit2.cpp @@ -179,6 +179,184 @@ Status HfDownloader::RemoveReadonlyFileAttributeFromDir(const std::string& direc return StatusCode::OK; } +Status HfDownloader::CheckRepositoryStatus() { + git_repository *repo = NULL; + int error = git_repository_open_ext(&repo, this->downloadPath.c_str(), 0, NULL); + if (error < 0) { + const git_error *err = git_error_last(); + if (err) + SPDLOG_ERROR("Repository open failed: {} {}", err->klass, err->message); + else + SPDLOG_ERROR("Repository open failed: {}", error); + if (repo) git_repository_free(repo); + + return StatusCode::HF_GIT_STATUS_FAILED; + } + // HEAD state info + bool is_detached = git_repository_head_detached(repo) == 1; + bool is_unborn = git_repository_head_unborn(repo) == 1; + + // Collect status (staged/unstaged/untracked) + git_status_options opts = GIT_STATUS_OPTIONS_INIT; + + opts.show = GIT_STATUS_SHOW_INDEX_AND_WORKDIR; + opts.flags = GIT_STATUS_OPT_INCLUDE_UNTRACKED // include untracked files // | GIT_STATUS_OPT_RENAMES_HEAD_TO_INDEX // detect renames HEAD->index - not required currently and impacts performance + | GIT_STATUS_OPT_SORT_CASE_SENSITIVELY; + + + git_status_list* status_list = nullptr; + error = git_status_list_new(&status_list, repo, &opts); + if (error != 0) { + return StatusCode::HF_GIT_STATUS_FAILED; + } + + size_t staged = 0, unstaged = 0, untracked = 0, conflicted = 0; + const size_t n = git_status_list_entrycount(status_list); // iterate entries + for (size_t i = 0; i < n; ++i) { + const git_status_entry* e = git_status_byindex(status_list, i); + unsigned s = e->status; + + // Staged (index) changes + if (s & (GIT_STATUS_INDEX_NEW | + GIT_STATUS_INDEX_MODIFIED| + GIT_STATUS_INDEX_DELETED | + GIT_STATUS_INDEX_RENAMED | + GIT_STATUS_INDEX_TYPECHANGE)) + ++staged; + + // Unstaged (workdir) changes + if (s & (GIT_STATUS_WT_MODIFIED | + GIT_STATUS_WT_DELETED | + GIT_STATUS_WT_RENAMED | + GIT_STATUS_WT_TYPECHANGE)) + ++unstaged; + + // Untracked + if (s & GIT_STATUS_WT_NEW) + ++untracked; + + // libgit2 will also flag conflicted entries via status/diff machinery + if (s & GIT_STATUS_CONFLICTED) + ++conflicted; + } + + std::stringstream ss; + ss << "HEAD state : " + << (is_unborn ? "unborn (no commits)" : (is_detached ? "detached" : "attached")) + << "\n"; + ss << "Staged changes : " << staged << "\n"; + ss << "Unstaged changes: " << unstaged << "\n"; + ss << "Untracked files : " << untracked << "\n"; + if (conflicted) ss << " (" << conflicted << " paths flagged)"; + + SPDLOG_DEBUG(ss.str()); + git_status_list_free(status_list); + + if (is_unborn || is_detached || staged || unstaged || untracked || conflicted) { + return StatusCode::HF_GIT_STATUS_UNCLEAN; + } + return StatusCode::OK; +} + +static int print_changed_and_untracked(git_repository *repo) { + int error = 0; + git_status_list *statuslist = NULL; + + git_status_options opts; + error = git_status_options_init(&opts, GIT_STATUS_OPTIONS_VERSION); + if (error < 0) return error; + + // Choose what to include + opts.show = GIT_STATUS_SHOW_INDEX_AND_WORKDIR; // consider both index and working dir + opts.flags = + GIT_STATUS_OPT_INCLUDE_UNTRACKED | // include untracked files + GIT_STATUS_OPT_RECURSE_UNTRACKED_DIRS | // recurse into untracked dirs + GIT_STATUS_OPT_INCLUDE_IGNORED | // (optional) include ignored if you want to see them + GIT_STATUS_OPT_RENAMES_HEAD_TO_INDEX | // detect renames in index + GIT_STATUS_OPT_RENAMES_INDEX_TO_WORKDIR | // detect renames in workdir + GIT_STATUS_OPT_SORT_CASE_SENSITIVELY; // stable ordering + + // If you want to limit to certain paths/patterns, set opts.pathspec here. + + if ((error = git_status_list_new(&statuslist, repo, &opts)) < 0) + return error; + + size_t count = git_status_list_entrycount(statuslist); + for (size_t i = 0; i < count; i++) { + const git_status_entry *e = git_status_byindex(statuslist, i); + if (!e) continue; + + unsigned int s = e->status; + + // Consider “changed” as anything that’s not current in HEAD/INDEX/WT: + // You can tailor this to your exact definition. + int is_untracked = + (s & GIT_STATUS_WT_NEW) != 0; // working tree new (untracked) + int is_workdir_changed = + (s & (GIT_STATUS_WT_MODIFIED | + GIT_STATUS_WT_DELETED | + GIT_STATUS_WT_RENAMED | + GIT_STATUS_WT_TYPECHANGE)) != 0; + int is_index_changed = + (s & (GIT_STATUS_INDEX_NEW | + GIT_STATUS_INDEX_MODIFIED | + GIT_STATUS_INDEX_DELETED | + GIT_STATUS_INDEX_RENAMED | + GIT_STATUS_INDEX_TYPECHANGE)) != 0; + + if (!(is_untracked || is_workdir_changed || is_index_changed)) + continue; + + // Prefer the most relevant delta for the path + const git_diff_delta *delta = NULL; + if (is_workdir_changed && e->index_to_workdir) + delta = e->index_to_workdir; + else if (is_index_changed && e->head_to_index) + delta = e->head_to_index; + else if (is_untracked && e->index_to_workdir) + delta = e->index_to_workdir; + + if (!delta) continue; + + // For renames, old_file and new_file may differ; typically you want new_file.path + const char *path = delta->new_file.path ? delta->new_file.path + : delta->old_file.path; + + // Print or collect the filename + SPDLOG_INFO("is_untracked {} is_workdir_changed {} is_index_changed {} File {} ", is_untracked, is_workdir_changed, is_index_changed, path); + } + + git_status_list_free(statuslist); + return 0; +} + +int HfDownloader::CheckRepositoryForResume() { + git_repository *repo = NULL; + int error = git_repository_open_ext(&repo, this->downloadPath.c_str(), 0, NULL); + if (error < 0) { + const git_error *err = git_error_last(); + if (err) + SPDLOG_ERROR("Repository open failed: {} {}", err->klass, err->message); + else + SPDLOG_ERROR("Repository open failed: {}", error); + if (repo) git_repository_free(repo); + + return error; + } + + error = print_changed_and_untracked(repo); + if (error < 0) { + const git_error *err = git_error_last(); + if (err) + SPDLOG_ERROR("Print changed files failed: {} {}", err->klass, err->message); + else + SPDLOG_ERROR("Print changed files failed: {}", error); + } + + if (repo) git_repository_free(repo); + return error; +} + Status HfDownloader::downloadModel() { if (FileSystem::isPathEscaped(this->downloadPath)) { SPDLOG_ERROR("Path {} escape with .. is forbidden.", this->downloadPath); @@ -187,6 +365,9 @@ Status HfDownloader::downloadModel() { // Repository exists and we do not want to overwrite if (std::filesystem::is_directory(this->downloadPath) && !this->overwriteModels) { + CheckRepositoryForResume(); + + std::cout << "Path already exists on local filesystem. Skipping download to path: " << this->downloadPath << std::endl; return StatusCode::OK; } @@ -231,6 +412,12 @@ Status HfDownloader::downloadModel() { git_repository_free(cloned_repo); } + SPDLOG_DEBUG("Checking repository status."); + status = CheckRepositoryStatus(); + if (!status.ok()) { + return status; + } + // libgit2 clone sets readonly attributes status = RemoveReadonlyFileAttributeFromDir(this->downloadPath); if (!status.ok()) { diff --git a/src/pull_module/libgit2.hpp b/src/pull_module/libgit2.hpp index 943f3cf725..755a84e89e 100644 --- a/src/pull_module/libgit2.hpp +++ b/src/pull_module/libgit2.hpp @@ -62,5 +62,7 @@ class HfDownloader : public IModelDownloader { std::string GetRepositoryUrlWithPassword(); bool CheckIfProxySet(); Status RemoveReadonlyFileAttributeFromDir(const std::string& directoryPath); + Status CheckRepositoryStatus(); + int CheckRepositoryForResume(); }; } // namespace ovms diff --git a/src/status.cpp b/src/status.cpp index 97a92b9d30..b83b4f7a45 100644 --- a/src/status.cpp +++ b/src/status.cpp @@ -348,6 +348,9 @@ const std::unordered_map Status::statusMessageMap = { {StatusCode::HF_RUN_OPTIMUM_CLI_EXPORT_FAILED, "Failed to run optimum-cli export command"}, {StatusCode::HF_RUN_CONVERT_TOKENIZER_EXPORT_FAILED, "Failed to run convert-tokenizer export command"}, {StatusCode::HF_GIT_CLONE_FAILED, "Failed in libgit2 execution of clone method"}, + {StatusCode::HF_GIT_STATUS_FAILED, "Failed in libgit2 execution of status method"}, + {StatusCode::HF_GIT_STATUS_UNCLEAN, "Unclean status detected in libgit2 cloned repository"}, + {StatusCode::PARTIAL_END, "Request has finished and no further communication is needed"}, {StatusCode::NONEXISTENT_PATH, "Nonexistent path"}, diff --git a/src/status.hpp b/src/status.hpp index 18a2b093b5..2f72532cfd 100644 --- a/src/status.hpp +++ b/src/status.hpp @@ -360,6 +360,8 @@ enum class StatusCode { HF_RUN_OPTIMUM_CLI_EXPORT_FAILED, HF_RUN_CONVERT_TOKENIZER_EXPORT_FAILED, HF_GIT_CLONE_FAILED, + HF_GIT_STATUS_FAILED, + HF_GIT_STATUS_UNCLEAN, PARTIAL_END, NONEXISTENT_PATH, From 7dcf65916712243492c696defb9f0c7574619889 Mon Sep 17 00:00:00 2001 From: rasapala Date: Fri, 27 Feb 2026 11:52:26 +0100 Subject: [PATCH 04/33] Works on windows --- third_party/libgit2/lfs.patch | 110 +++++++++++++++++++++++++--------- 1 file changed, 81 insertions(+), 29 deletions(-) diff --git a/third_party/libgit2/lfs.patch b/third_party/libgit2/lfs.patch index 9d09b48943..02a81c57e4 100644 --- a/third_party/libgit2/lfs.patch +++ b/third_party/libgit2/lfs.patch @@ -312,10 +312,10 @@ index 58cb4b424..00ddee9f3 100644 git_writestream **out, diff --git a/src/libgit2/lfs_filter.c b/src/libgit2/lfs_filter.c new file mode 100644 -index 000000000..3b42a7ba4 +index 000000000..5a005f903 --- /dev/null +++ b/src/libgit2/lfs_filter.c -@@ -0,0 +1,603 @@ +@@ -0,0 +1,655 @@ +/* +/ Copyright 2025 Intel Corporation +/ @@ -337,6 +337,7 @@ index 000000000..3b42a7ba4 + +#include +#include "git2/sys/filter.h" ++#include "oid.h" +#include "filter.h" +#include "str.h" +#include "repository.h" @@ -360,27 +361,49 @@ index 000000000..3b42a7ba4 + size_t number = strtoull(buffer, &endptr, 10); + + if (errno == ERANGE) { -+ fprintf(stderr, "Conversion error\n"); ++ fprintf(stderr, "\n[ERROR] Conversion error\n"); + } + if (endptr == buffer) { -+ fprintf(stderr, "No digits were found\n"); ++ fprintf(stderr, "\n[ERROR] No digits were found\n"); + } else if (*endptr != '\0') { -+ fprintf(stderr, "Additional characters after number: %s\n", endptr); ++ fprintf(stderr, "\n[ERROR] Additional characters after number: %s\n", endptr); + } + + return number; +} + -+char *append_char_to_buffer(char *existingBuffer, char additionalChar) ++/** ++ * Appends a C-string `suffix` to `existingBuffer` by allocating a new buffer. ++ * The original `existingBuffer` is not modified. ++ * ++ * Returns: ++ * - Newly allocated buffer containing the concatenation, or ++ * - NULL on allocation failure or if inputs are invalid. ++ * ++ * Note: Caller is responsible for freeing the returned buffer. ++ */ ++char *append_cstr_to_buffer(const char *existingBuffer, const char *suffix) +{ ++ if (existingBuffer == NULL || suffix == NULL) { ++ return NULL; ++ } ++ + size_t existingLength = strlen(existingBuffer); -+ char *newBuffer = (char *)malloc((existingLength + 2) * sizeof(char)); ++ size_t suffixLength = strlen(suffix); ++ ++ // +1 for the null terminator ++ size_t newSize = existingLength + suffixLength + 1; ++ ++ char *newBuffer = (char *)malloc(newSize); + if (newBuffer == NULL) { + return NULL; + } -+ strcpy(newBuffer, existingBuffer); -+ newBuffer[existingLength] = additionalChar; -+ newBuffer[existingLength + 1] = '\0'; ++ ++ // Copy existing and then append suffix ++ memcpy(newBuffer, existingBuffer, existingLength); ++ memcpy(newBuffer + existingLength, suffix, suffixLength); ++ newBuffer[newSize - 1] = '\0'; ++ + return newBuffer; +} + @@ -412,6 +435,17 @@ index 000000000..3b42a7ba4 + return -1; +} + ++void print_src_oid(const git_filter_source *src) ++{ ++ const git_oid *oid = git_filter_source_id(src); ++ ++ if (oid) { ++ printf("\nsrc->git_oid %s\n", git_oid_tostr_s(oid)); ++ } else { ++ printf("\nsrc has no OID (e.g., not a blob-backed source or unavailable)\n"); ++ } ++} ++ +static int lfs_insert_id( + git_str *to, const git_str *from, const git_filter_source *src, void** payload) +{ @@ -427,18 +461,31 @@ index 000000000..3b42a7ba4 + + const char *obj_regexp = "\noid sha256:(.*)\n"; + const char *size_regexp = "\nsize (.*)\n"; -+ if (get_lfs_info_match(&lfs_oid, obj_regexp) < 0) ++ ++ print_src_oid(src); ++ if (get_lfs_info_match(&lfs_oid, obj_regexp) < 0) { ++ fprintf(stderr,"\n[ERROR] failure, cannot find lfs oid in: %s\n", ++ lfs_oid.ptr); + return -1; ++ } + -+ if (get_lfs_info_match(&lfs_size, size_regexp) < 0) ++ if (get_lfs_info_match(&lfs_size, size_regexp) < 0) { ++ fprintf(stderr, ++ "\n[ERROR] failure, cannot find lfs size in: %s\n", ++ lfs_size.ptr); + return -1; ++ } + + git_repository *repo = git_filter_source_repo(src); + const char *path = git_filter_source_path(src); + + git_str full_path = GIT_STR_INIT; -+ if (git_repository_workdir_path(&full_path, repo, path) < 0) ++ if (git_repository_workdir_path(&full_path, repo, path) < 0) { ++ fprintf(stderr, ++ "\n[ERROR] failure, cannot get repository path: %s\n", ++ path); + return -1; ++ } + + size_t workdir_size = strlen(git_repository_workdir(repo)); + @@ -623,7 +670,7 @@ index 000000000..3b42a7ba4 + /* open file for writing */ + out->stream = fopen(out->filename, "wb"); + if (!out->stream) { -+ fprintf(stderr, "failure, cannot open file to write: %s\n", ++ fprintf(stderr, "\n[ERROR] failure, cannot open file to write: %s\n", + out->filename); + return 0; /* failure, cannot open file to write */ + } @@ -690,11 +737,11 @@ index 000000000..3b42a7ba4 +{ + GIT_UNUSED(self); + if (!payload) { -+ fprintf(stderr, "lfs payload not initialized"); ++ fprintf(stderr, "\n[ERROR] lfs payload not initialized\n"); + return; + } + struct lfs_attrs *la = (struct lfs_attrs *)payload; -+ char *tmp_out_file = append_char_to_buffer(la->full_path, '2'); ++ char *tmp_out_file = append_cstr_to_buffer(la->full_path, "lfs_part"); + + CURL *info_curl,*dl_curl; + CURLcode res = CURLE_OK; @@ -709,7 +756,7 @@ index 000000000..3b42a7ba4 + &lfs_info_url, '.', + la->url, + "git/info/lfs/objects/batch") < 0) { -+ fprintf(stderr, "failed to create url '%s'", ++ fprintf(stderr, "\n[ERROR] failed to create url '%s'\n", + la->full_path); + goto on_error; + } @@ -726,7 +773,7 @@ index 000000000..3b42a7ba4 + CURL_SETOPT(curl_easy_setopt(info_curl, CURLOPT_URL, lfs_info_url.ptr)); + + if (status != CURLE_OK) { -+ fprintf(stderr, "curl_easy_setopt() failed: %s\n", curl_easy_strerror(status)); ++ fprintf(stderr, "\n[ERROR] curl_easy_setopt() failed: %s\n", curl_easy_strerror(status)); + goto info_cleaup; + } + git_str lfs_info_data = GIT_STR_INIT; @@ -739,7 +786,7 @@ index 000000000..3b42a7ba4 + ",\"size\":", + la->lfs_size, + "}]}" ) < 0) { -+ fprintf(stderr, "failed to create url '%s'", ++ fprintf(stderr, "\n[ERROR] failed to create url '%s'\n", + la->full_path); + /* always cleanup */ + curl_easy_cleanup(info_curl); @@ -759,14 +806,14 @@ index 000000000..3b42a7ba4 + CURL_SETOPT(curl_easy_setopt(info_curl, CURLOPT_WRITEDATA, (void *)&response)); + + if (status != CURLE_OK) { -+ fprintf(stderr, "curl_easy_setopt() failed: %s\n", curl_easy_strerror(status)); ++ fprintf(stderr, "\n[ERROR] curl_easy_setopt() failed: %s\n", curl_easy_strerror(status)); + goto info_cleaup; + } + /* Perform the request, res gets the return code */ + res = curl_easy_perform(info_curl); + /* Check for errors */ + if (res != CURLE_OK) { -+ fprintf(stderr, "curl_easy_perform() failed: %s\n", ++ fprintf(stderr, "\n[ERROR] curl_easy_perform() failed: %s\n", + curl_easy_strerror(res)); + /* always cleanup */ + curl_easy_cleanup(info_curl); @@ -817,7 +864,7 @@ index 000000000..3b42a7ba4 + CURL_SETOPT(curl_easy_setopt(dl_curl, CURLOPT_XFERINFODATA, &progress_d)); + + if (status != CURLE_OK) { -+ fprintf(stderr, "curl_easy_setopt() failed: %s\n", curl_easy_strerror(status)); ++ fprintf(stderr, "\n[ERROR] curl_easy_setopt() failed: %s\n", curl_easy_strerror(status)); + curl_easy_cleanup(dl_curl); + goto on_error; + } @@ -833,10 +880,10 @@ index 000000000..3b42a7ba4 + + if (resume_from == -1) { + fprintf(stderr, -+ "curl_easy_perform() failed with transferred a partial file error and server does not support range/resume.\n"); ++ "\n[ERROR] curl_easy_perform() failed with transferred a partial file error and server does not support range/resume.\n"); + } else { + fprintf(stderr, -+ "curl_easy_perform() failed with transferred a partial file error and trying to resume\n"); ++ "\n[ERROR] curl_easy_perform() failed with transferred a partial file error and trying to resume\n"); + curl_off_t offset = 0; + if (ftpfile.stream) { + fseek(ftpfile.stream, 0, SEEK_END); @@ -862,12 +909,15 @@ index 000000000..3b42a7ba4 + + /* Check for errors */ + if (res != CURLE_OK) { -+ fprintf(stderr, "curl_easy_perform() failed: %s\n", ++ fprintf(stderr, "\n[ERROR] curl_easy_perform() failed: %s\n", + curl_easy_strerror(res)); + if (ftpfile.stream) + fclose(ftpfile.stream); + /* always cleanup */ + curl_easy_cleanup(dl_curl); ++ ++ /* Add partial file status */ ++ + goto on_error; + } + @@ -879,20 +929,22 @@ index 000000000..3b42a7ba4 + + /* Remove lfs file and rename downloaded file to oryginal lfs filename */ + if (p_unlink(la->full_path) < 0) { -+ fprintf(stderr, "failed to delete file '%s'", la->full_path); ++ fprintf(stderr, "\n[ERROR] failed to delete file '%s'\n", la->full_path); + goto on_error; + } + + if (p_rename(tmp_out_file, la->full_path) < 0) { -+ fprintf(stderr, "failed to rename file to '%s'", la->full_path); ++ fprintf(stderr, "\n[ERROR] failed to rename file to '%s'\n", la->full_path); + goto on_error; + } ++ free(tmp_out_file); + git__free(payload); + return; + -+on_error: ++ on_error: ++ free(tmp_out_file); + git__free(payload); -+ fprintf(stderr, "LFS download failed for file %s\n", la->full_path); ++ fprintf(stderr, "\n[ERROR] LFS download failed for file %s\n", la->full_path); + return; +} + From 1ea3962b947ea2189073699bd57443e869a935a7 Mon Sep 17 00:00:00 2001 From: rasapala Date: Fri, 27 Feb 2026 11:57:15 +0100 Subject: [PATCH 05/33] Fix lin --- third_party/libgit2/lfs.patch | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/third_party/libgit2/lfs.patch b/third_party/libgit2/lfs.patch index 02a81c57e4..3b2e8a5024 100644 --- a/third_party/libgit2/lfs.patch +++ b/third_party/libgit2/lfs.patch @@ -312,7 +312,7 @@ index 58cb4b424..00ddee9f3 100644 git_writestream **out, diff --git a/src/libgit2/lfs_filter.c b/src/libgit2/lfs_filter.c new file mode 100644 -index 000000000..5a005f903 +index 000000000..7c1111c34 --- /dev/null +++ b/src/libgit2/lfs_filter.c @@ -0,0 +1,655 @@ @@ -391,7 +391,7 @@ index 000000000..5a005f903 + size_t existingLength = strlen(existingBuffer); + size_t suffixLength = strlen(suffix); + -+ // +1 for the null terminator ++ /* +1 for the null terminator */ + size_t newSize = existingLength + suffixLength + 1; + + char *newBuffer = (char *)malloc(newSize); @@ -399,7 +399,7 @@ index 000000000..5a005f903 + return NULL; + } + -+ // Copy existing and then append suffix ++ /* Copy existing and then append suffix */ + memcpy(newBuffer, existingBuffer, existingLength); + memcpy(newBuffer + existingLength, suffix, suffixLength); + newBuffer[newSize - 1] = '\0'; From 3d7a6090a77242f3f9a508780327c31f3d839bd4 Mon Sep 17 00:00:00 2001 From: Rafal Sapala Date: Mon, 2 Mar 2026 18:13:15 +0100 Subject: [PATCH 06/33] Debug --- src/pull_module/libgit2.cpp | 473 +++++++++++++++++++++++++++++++++++- 1 file changed, 472 insertions(+), 1 deletion(-) diff --git a/src/pull_module/libgit2.cpp b/src/pull_module/libgit2.cpp index 395d461a7f..423320b1af 100644 --- a/src/pull_module/libgit2.cpp +++ b/src/pull_module/libgit2.cpp @@ -45,6 +45,7 @@ #endif namespace ovms { +namespace fs = std::filesystem; // Callback for clone authentication - will be used when password is not set in repo_url // Does not work with LFS download as it requires additional authentication when password is not set in repository url @@ -330,6 +331,339 @@ static int print_changed_and_untracked(git_repository *repo) { return 0; } +#define CHECK(call) do { \ + int _err = (call); \ + if (_err < 0) { \ + const git_error *e = git_error_last(); \ + fprintf(stderr, "Error %d: %s (%s:%d)\n", _err, e && e->message ? e->message : "no message", __FILE__, __LINE__); \ + return; \ + } \ +} while (0) + +// Fetch from remote and update FETCH_HEAD +static void do_fetch(git_repository *repo, const char *remote_name, const char *proxy) +{ + git_remote *remote = NULL; + git_fetch_options fetch_opts = GIT_FETCH_OPTIONS_INIT; + + fetch_opts.prune = GIT_FETCH_PRUNE_UNSPECIFIED; + fetch_opts.update_fetchhead = 1; + fetch_opts.download_tags = GIT_REMOTE_DOWNLOAD_TAGS_ALL; + fetch_opts.callbacks = (git_remote_callbacks) GIT_REMOTE_CALLBACKS_INIT; + fetch_opts.callbacks.credentials = cred_acquire_cb; + if (proxy) { + fetch_opts.proxy_opts.type = GIT_PROXY_SPECIFIED; + fetch_opts.proxy_opts.url = proxy; + } + + CHECK(git_remote_lookup(&remote, repo, remote_name)); + + printf("Fetching from %s...\n", remote_name); + CHECK(git_remote_fetch(remote, NULL, &fetch_opts, NULL)); + + // Optional: update remote-tracking branches' default refspec tips + // (git_remote_fetch already updates tips if update_fetchhead=1; explicit + // update_tips is not required in recent libgit2 versions.) + + git_remote_free(remote); +} + +// Fast-forward the local branch to target OID +static void do_fast_forward(git_repository *repo, + git_reference *local_branch, + const git_oid *target_oid) +{ + git_object *target = NULL; + git_checkout_options co_opts = GIT_CHECKOUT_OPTIONS_INIT; + git_reference *updated_ref = NULL; + + CHECK(git_object_lookup(&target, repo, target_oid, GIT_OBJECT_COMMIT)); + + // Update the branch reference to point to the target commit + CHECK(git_reference_set_target(&updated_ref, local_branch, target_oid, "Fast-forward")); + + // Make sure HEAD points to that branch (it normally already does) + CHECK(git_repository_set_head(repo, git_reference_name(updated_ref))); + + // Checkout files to match the target tree + co_opts.checkout_strategy = GIT_CHECKOUT_SAFE; + CHECK(git_checkout_tree(repo, target, &co_opts)); + + printf("Fast-forwarded %s to %s\n", + git_reference_shorthand(local_branch), + git_oid_tostr_s(target_oid)); + + git_object_free(target); + git_reference_free(updated_ref); +} + +// Perform a normal merge and create a merge commit if no conflicts +static void do_normal_merge(git_repository *repo, + git_reference *local_branch, + git_annotated_commit *their_head) +{ + git_merge_options merge_opts = GIT_MERGE_OPTIONS_INIT; + git_checkout_options co_opts = GIT_CHECKOUT_OPTIONS_INIT; + + merge_opts.file_favor = GIT_MERGE_FILE_FAVOR_NORMAL; + co_opts.checkout_strategy = GIT_CHECKOUT_SAFE | GIT_CHECKOUT_RECREATE_MISSING; + + const git_annotated_commit *their_heads[1] = { their_head }; + + printf("Merging...\n"); + CHECK(git_merge(repo, their_heads, 1, &merge_opts, &co_opts)); + + // Check for conflicts + git_index *index = NULL; + CHECK(git_repository_index(&index, repo)); + + if (git_index_has_conflicts(index)) { + printf("Merge has conflicts. Please resolve them and create the merge commit manually.\n"); + git_index_free(index); + return; // Leave repository in merging state + } + + // Write index to tree + git_oid tree_oid; + CHECK(git_index_write_tree(&tree_oid, index)); + CHECK(git_index_write(index)); + git_index_free(index); + + git_tree *tree = NULL; + CHECK(git_tree_lookup(&tree, repo, &tree_oid)); + + // Prepare signature (from config if available) + git_signature *sig = NULL; + int err = git_signature_default(&sig, repo); + if (err == GIT_ENOTFOUND || sig == NULL) { + // Fallback if user.name/email not set in config + CHECK(git_signature_now(&sig, "Your Name", "you@example.com")); + } else { + CHECK(err); + } + + // Get current HEAD (our) commit and their commit to be parents + git_reference *head = NULL, *resolved_branch = NULL; + CHECK(git_repository_head(&head, repo)); + CHECK(git_reference_resolve(&resolved_branch, head)); // ensure direct ref + + const git_oid *our_oid = git_reference_target(resolved_branch); + git_commit *our_commit = NULL; + git_commit *their_commit = NULL; + CHECK(git_commit_lookup(&our_commit, repo, our_oid)); + CHECK(git_commit_lookup(&their_commit, repo, git_annotated_commit_id(their_head))); + + const git_commit *parents[2] = { our_commit, their_commit }; + git_oid merge_commit_oid; + + // Create merge commit on the current branch ref + CHECK(git_commit_create(&merge_commit_oid, + repo, + git_reference_name(resolved_branch), + sig, sig, + NULL /* message_encoding */, + "Merge remote-tracking branch", + tree, + 2, parents)); + + printf("Created merge commit %s on %s\n", + git_oid_tostr_s(&merge_commit_oid), + git_reference_shorthand(resolved_branch)); + + // Cleanup + git_signature_free(sig); + git_tree_free(tree); + git_commit_free(our_commit); + git_commit_free(their_commit); + git_reference_free(head); + git_reference_free(resolved_branch); +} + +// Main pull routine: fetch + merge (fast-forward if possible) +static void pull(git_repository *repo, const char *remote_name, const char *proxy) +{ + // Ensure we are on a branch (not detached HEAD) + git_reference *head = NULL; + int head_res = git_repository_head(&head, repo); + // HEAD state info + bool is_detached = git_repository_head_detached(repo) == 1; + bool is_unborn = git_repository_head_unborn(repo) == 1; + if (is_unborn) { + fprintf(stderr, "Repository has no HEAD yet (unborn branch).\n"); + return; + } else if (is_detached) { + fprintf(stderr, "HEAD is detached; cannot pull safely. Checkout a branch first.\n"); + return; + } + CHECK(head_res); + + // Resolve symbolic HEAD to direct branch ref (refs/heads/…) + git_reference *local_branch = NULL; + CHECK(git_reference_resolve(&local_branch, head)); + + // Find the upstream tracking branch (refs/remotes//) + git_reference *upstream = NULL; + int up_ok = git_branch_upstream(&upstream, local_branch); + if (up_ok != 0 || upstream == NULL) { + fprintf(stderr, "Current branch has no upstream. Set it with:\n" + " git branch --set-upstream-to=%s/ \n", remote_name); + return; + } + + // Verify upstream belongs to the requested remote; not strictly required for fetch + // but we fetch from the chosen remote anyway. + do_fetch(repo, remote_name, proxy); + + // Prepare "their" commit as annotated commit from upstream + git_annotated_commit *their_head = NULL; + CHECK(git_annotated_commit_from_ref(&their_head, repo, upstream)); + + // Merge analysis + git_merge_analysis_t analysis; + git_merge_preference_t preference; + CHECK(git_merge_analysis(&analysis, &preference, repo, + (const git_annotated_commit **)&their_head, 1)); + + if (analysis & GIT_MERGE_ANALYSIS_UP_TO_DATE) { + printf("Already up to date.\n"); + } else if (analysis & GIT_MERGE_ANALYSIS_FASTFORWARD) { + const git_oid *target_oid = git_annotated_commit_id(their_head); + do_fast_forward(repo, local_branch, target_oid); + } else if (analysis & GIT_MERGE_ANALYSIS_NORMAL) { + do_normal_merge(repo, local_branch, their_head); + } else { + printf("No merge action taken (analysis=%u, preference=%u).\n", + (unsigned)analysis, (unsigned)preference); + } + + // Cleanup + git_annotated_commit_free(their_head); + git_reference_free(upstream); + git_reference_free(local_branch); + git_reference_free(head); +} + + +// Trim trailing '\r' (for CRLF files) and surrounding spaces +static inline void rtrimCrLfWhitespace(std::string& s) { + if (!s.empty() && s.back() == '\r') s.pop_back(); // remove trailing '\r' + while (!s.empty() && std::isspace(static_cast(s.back()))) s.pop_back(); // trailing ws + size_t i = 0; + while (i < s.size() && std::isspace(static_cast(s[i]))) ++i; // leading ws + if (i > 0) s.erase(0, i); +} + +// Case-insensitive substring search: returns true if 'needle' is found in 'hay' +static bool containsCaseInsensitive(const std::string& hay, const std::string& needle) { + auto toLower = [](std::string v) { + std::transform(v.begin(), v.end(), v.begin(), + [](unsigned char c){ return static_cast(std::tolower(c)); }); + return v; + }; + std::string hayLower = toLower(hay); + std::string needleLower = toLower(needle); + return hayLower.find(needleLower) != std::string::npos; +} + +// Read at most the first 3 lines of a file, with a per-line cap to avoid huge reads. +// Returns true if successful (even if <3 lines exist; vector will just be shorter). +static bool readFirstThreeLines(const fs::path& p, std::vector& outLines) { + outLines.clear(); + std::ifstream in(p, std::ios::in | std::ios::binary); + if (!in) return false; + + constexpr std::streamsize kMaxPerLine = 8192; + + std::string line; + line.reserve(static_cast(kMaxPerLine)); + for (int i = 0; i < 3 && in.good(); ++i) { + line.clear(); + std::streamsize count = 0; + char ch; + bool gotNewline = false; + while (count < kMaxPerLine && in.get(ch)) { + if (ch == '\n') { gotNewline = true; break; } + line.push_back(ch); + ++count; + } + // If we hit kMaxPerLine without encountering '\n', drain until newline to resync + if (count == kMaxPerLine && !gotNewline) { + while (in.get(ch)) { + if (ch == '\n') break; + } + } + + if (!in && line.empty()) { + // EOF with no data accumulated; if previous lines were read, that's fine. + break; + } + rtrimCrLfWhitespace(line); + outLines.push_back(line); + if (!in) break; // Handle EOF gracefully + } + return true; +} + +// Check if the first 3 lines contain required keywords in positional order: +// line1 -> "version", line2 -> "oid", line3 -> "size" (case-insensitive). +static bool fileHasLfsKeywordsFirst3Positional(const fs::path& p) { + std::error_code ec; + if (!fs::is_regular_file(p, ec)) return false; + + std::vector lines; + if (!readFirstThreeLines(p, lines)) return false; + + if (lines.size() < 3) return false; + + return containsCaseInsensitive(lines[0], "version") && + containsCaseInsensitive(lines[1], "oid") && + containsCaseInsensitive(lines[2], "size"); +} + + +// Helper: make path relative to base (best-effort, non-throwing). +static fs::path makeRelativeToBase(const fs::path& path, const fs::path& base) { + std::error_code ec; + // Try fs::relative first (handles canonical comparisons, may fail if on different roots) + fs::path rel = fs::relative(path, base, ec); + if (!ec && !rel.empty()) return rel; + + // Fallback: purely lexical relative (doesn't access filesystem) + rel = path.lexically_relative(base); + if (!rel.empty()) return rel; + + // Last resort: return filename only (better than absolute when nothing else works) + if (path.has_filename()) return path.filename(); + return path; +} + +// Find all files under 'directory' that satisfy the first-3-lines LFS keyword check. +std::vector findLfsLikeFiles(const std::string& directory, bool recursive = true) { + std::vector matches; + std::error_code ec; + + if (!fs::exists(directory, ec) || !fs::is_directory(directory, ec)) { + return matches; + } + + if (recursive) { + for (fs::recursive_directory_iterator it(directory, ec), end; !ec && it != end; ++it) { + const auto& p = it->path(); + if (fileHasLfsKeywordsFirst3Positional(p)) { + matches.push_back(makeRelativeToBase(p, directory)); + } + } + } else { + for (fs::directory_iterator it(directory, ec), end; !ec && it != end; ++it) { + const auto& p = it->path(); + if (fileHasLfsKeywordsFirst3Positional(p)) { + matches.push_back(makeRelativeToBase(p, directory)); + } + } + } + return matches; +} + int HfDownloader::CheckRepositoryForResume() { git_repository *repo = NULL; int error = git_repository_open_ext(&repo, this->downloadPath.c_str(), 0, NULL); @@ -357,6 +691,107 @@ int HfDownloader::CheckRepositoryForResume() { return error; } + +/* + * checkout_one_file: Check out a single path from a treeish (commit/ref) + * into the working directory, applying filters (smudge, EOL) just like clone. + * + * repo_path : filesystem path to the existing (non-bare) repository + * treeish : e.g., "HEAD", "origin/main", a full commit SHA, etc. + * path_in_repo : repo-relative path (e.g., "src/main.c") + * + * Returns 0 on success; <0 (libgit2 error code) on failure. + */ +void resumeLfsDownloadForFile(git_repository *repo, fs::path fileToResume, const std::string& repositoryPath) { + int error = 0; + + // Remove existing lfs pointer file from repository + std::string fullPath = FileSystem::joinPath({repositoryPath, fileToResume.string()}); + std::filesystem::path filePath(fullPath); + if (!std::filesystem::remove(filePath)) { + SPDLOG_ERROR("Removing lfs file pointer error {}", fullPath); + return; + } + + const char *path_in_repo = fileToResume.string().c_str(); + const char *treeish = "HEAD"; + git_object *target = NULL; + git_strarray paths = {0}; + git_checkout_options opts = GIT_CHECKOUT_OPTIONS_INIT; + opts.disable_filters = 0; // default; ensure you are NOT disabling filters + + if (git_repository_is_bare(repo)) { + SPDLOG_ERROR("Repository is bare; cannot checkout to working directory {}", fileToResume.string()); + error = GIT_EBAREREPO; + goto done; + } + + if ((error = git_revparse_single(&target, repo, treeish)) < 0) { + SPDLOG_ERROR("git_revparse_single failed {}", fileToResume.string()); + goto done; + } + + // Restrict checkout to a single path + paths.count = 1; + paths.strings = (char **)&path_in_repo; + + opts.paths = paths; + + // Strategy: SAFER defaults — apply filters, write new files, update existing file + // You can add GIT_CHECKOUT_FORCE if you want to overwrite conflicts. + // opts.checkout_strategy = GIT_CHECKOUT_SAFE | GIT_CHECKOUT_DISABLE_PATHSPEC_MATCH; + opts.checkout_strategy = GIT_CHECKOUT_FORCE; // This makes sure BLOB update with filter is called + // This actually writes the filtered content to the working directory + error = git_checkout_tree(repo, target, &opts); + if (error < 0) { + SPDLOG_ERROR("git_checkout_tree failed {}", fileToResume.string()); + } + +done: + if (target) git_object_free(target); + return; +} + +/* Does not work +void resumeLfsDownloadForFile(git_repository *repo, fs::path fileToResume) { + git_object *obj = NULL; + git_tree *tree = NULL; + git_tree_entry *entry = NULL; + git_blob *blob = NULL; + git_buf out = GIT_BUF_INIT; + // Configure filter behavior + git_blob_filter_options opts = GIT_BLOB_FILTER_OPTIONS_INIT; + // Choose direction: + // GIT_BLOB_FILTER_TO_WORKTREE : apply smudge (as if writing to working tree) + // GIT_BLOB_FILTER_TO_ODB : apply clean (as if writing to ODB) + // opts.flags = GIT_FILTER_TO_WORKTREE; + + const char *file_path_in_repo = fileToResume.string().c_str(); // relative to repo root + + // Resolve HEAD tree + CHECK(git_revparse_single(&obj, repo, "HEAD^{tree}") != 0); + tree = (git_tree *)obj; + + // Find the tree entry and get the blob + CHECK(git_tree_entry_bypath(&entry, tree, file_path_in_repo) != 0); + CHECK(git_tree_entry_type(entry) != GIT_OBJECT_BLOB); + + CHECK(git_blob_lookup(&blob, repo, git_tree_entry_id(entry)) != 0); + + // Apply filters based on .gitattributes for this path + CHECK(git_blob_filter(&out, blob, file_path_in_repo, &opts) != 0); + + // out.ptr now contains the filtered content + fwrite(out.ptr, 1, out.size, stdout); + + git_buf_dispose(&out); + if (blob) git_blob_free(blob); + if (entry) git_tree_entry_free(entry); + if (tree) git_tree_free(tree); + if (obj) git_object_free(obj); + return; +} */ + Status HfDownloader::downloadModel() { if (FileSystem::isPathEscaped(this->downloadPath)) { SPDLOG_ERROR("Path {} escape with .. is forbidden.", this->downloadPath); @@ -365,8 +800,44 @@ Status HfDownloader::downloadModel() { // Repository exists and we do not want to overwrite if (std::filesystem::is_directory(this->downloadPath) && !this->overwriteModels) { - CheckRepositoryForResume(); + auto matches = findLfsLikeFiles(this->downloadPath, true); + + if (matches.empty()) { + std::cout << "No files with LFS-like keywords in the first 3 lines were found.\n"; + } else { + std::cout << "Found " << matches.size() << " matching file(s):\n"; + for (const auto& p : matches) { + std::cout << " " << p.string() << "\n"; + } + } + git_repository *repo = NULL; + int error = git_repository_open_ext(&repo, this->downloadPath.c_str(), 0, NULL); + if (error < 0) { + const git_error *err = git_error_last(); + if (err) + SPDLOG_ERROR("Repository open failed: {} {}", err->klass, err->message); + else + SPDLOG_ERROR("Repository open failed: {}", error); + if (repo) git_repository_free(repo); + + std::cout << "Path already exists on local filesystem. And is not a git repository: " << this->downloadPath << std::endl; + return StatusCode::HF_GIT_CLONE_FAILED; + } + + for (const auto& p : matches) { + std::cout << " Resuming " << p.string() << "\n"; + resumeLfsDownloadForFile(repo, p, this->downloadPath); + } + + // Use proxy + if (CheckIfProxySet()) { + SPDLOG_DEBUG("Download using https_proxy settings"); + //pull(repo, "origin", this->httpProxy.c_str()); + } else { + SPDLOG_DEBUG("Download with https_proxy not set"); + pull(repo, "origin", nullptr); + } std::cout << "Path already exists on local filesystem. Skipping download to path: " << this->downloadPath << std::endl; return StatusCode::OK; From 417b95c65c4f28570e609b2df79f6a8fe523ea42 Mon Sep 17 00:00:00 2001 From: Rafal Sapala Date: Tue, 3 Mar 2026 16:12:48 +0100 Subject: [PATCH 07/33] Blob filter works --- src/pull_module/libgit2.cpp | 102 ++++++++++++++++++++++++++++++++---- 1 file changed, 92 insertions(+), 10 deletions(-) diff --git a/src/pull_module/libgit2.cpp b/src/pull_module/libgit2.cpp index 423320b1af..8b3944e846 100644 --- a/src/pull_module/libgit2.cpp +++ b/src/pull_module/libgit2.cpp @@ -702,7 +702,7 @@ int HfDownloader::CheckRepositoryForResume() { * * Returns 0 on success; <0 (libgit2 error code) on failure. */ -void resumeLfsDownloadForFile(git_repository *repo, fs::path fileToResume, const std::string& repositoryPath) { +void resumeLfsDownloadForFile2(git_repository *repo, fs::path fileToResume, const std::string& repositoryPath) { int error = 0; // Remove existing lfs pointer file from repository @@ -714,7 +714,8 @@ void resumeLfsDownloadForFile(git_repository *repo, fs::path fileToResume, const } const char *path_in_repo = fileToResume.string().c_str(); - const char *treeish = "HEAD"; + // TODO: make sure we are on 'origin/main'. + const char *treeish = "origin/main^{tree}"; git_object *target = NULL; git_strarray paths = {0}; git_checkout_options opts = GIT_CHECKOUT_OPTIONS_INIT; @@ -740,7 +741,7 @@ void resumeLfsDownloadForFile(git_repository *repo, fs::path fileToResume, const // Strategy: SAFER defaults — apply filters, write new files, update existing file // You can add GIT_CHECKOUT_FORCE if you want to overwrite conflicts. // opts.checkout_strategy = GIT_CHECKOUT_SAFE | GIT_CHECKOUT_DISABLE_PATHSPEC_MATCH; - opts.checkout_strategy = GIT_CHECKOUT_FORCE; // This makes sure BLOB update with filter is called + opts.checkout_strategy = GIT_CHECKOUT_FORCE | GIT_CHECKOUT_SAFE; // This makes sure BLOB update with filter is called // This actually writes the filtered content to the working directory error = git_checkout_tree(repo, target, &opts); if (error < 0) { @@ -752,8 +753,7 @@ void resumeLfsDownloadForFile(git_repository *repo, fs::path fileToResume, const return; } -/* Does not work -void resumeLfsDownloadForFile(git_repository *repo, fs::path fileToResume) { +void resumeLfsDownloadForFile1(git_repository *repo, const char *file_path_in_repo) { git_object *obj = NULL; git_tree *tree = NULL; git_tree_entry *entry = NULL; @@ -766,10 +766,8 @@ void resumeLfsDownloadForFile(git_repository *repo, fs::path fileToResume) { // GIT_BLOB_FILTER_TO_ODB : apply clean (as if writing to ODB) // opts.flags = GIT_FILTER_TO_WORKTREE; - const char *file_path_in_repo = fileToResume.string().c_str(); // relative to repo root - // Resolve HEAD tree - CHECK(git_revparse_single(&obj, repo, "HEAD^{tree}") != 0); + CHECK(git_revparse_single(&obj, repo, "origin/main^{tree}") != 0); tree = (git_tree *)obj; // Find the tree entry and get the blob @@ -790,7 +788,83 @@ void resumeLfsDownloadForFile(git_repository *repo, fs::path fileToResume) { if (tree) git_tree_free(tree); if (obj) git_object_free(obj); return; -} */ +} + + +static int on_notify( + git_checkout_notify_t why, const char *path, + const git_diff_file *baseline, const git_diff_file *target, const git_diff_file *workdir, + void *payload) +{ + (void)baseline; (void)target; (void)workdir; (void)payload; + fprintf(stderr, "[checkout notify] why=%u path=%s\n", why, path ? path : "(null)"); + return 0; // non-zero would cancel +} + +void checkout_one_from_origin_master(git_repository *repo, const char *path_rel) { + int err = 0; + git_object *commitobj = NULL; + git_commit *commit = NULL; + git_tree *tree = NULL; + git_tree_entry *te = NULL; + git_diff *diff = NULL; + git_diff_options diffopts = GIT_DIFF_OPTIONS_INIT; + git_checkout_options opts = GIT_CHECKOUT_OPTIONS_INIT; + + + printf("Path to resume '%s' \n", path_rel); + /* 1) Resolve origin/master to a commit, then get its tree */ + if ((err = git_revparse_single(&commitobj, repo, "refs/remotes/origin/main^{commit}")) < 0) return; + commit = (git_commit *)commitobj; + if ((err = git_commit_tree(&tree, commit)) < 0) return; + + /* 2) Sanity-check: does the path exist in the target tree? */ + if ((err = git_tree_entry_bypath(&te, tree, path_rel)) == GIT_ENOTFOUND) { + fprintf(stderr, "Path '%s' not found in origin/main\n", path_rel); + err = 0; return; // nothing to do + } else if (err < 0) { + return; + } + git_tree_entry_free(te); te = NULL; + + /* 3) Diff target tree -> workdir (with index) for the one path */ + diffopts.pathspec.count = 1; + diffopts.pathspec.strings = (char **)&path_rel; + if ((err = git_diff_tree_to_workdir_with_index(&diff, repo, tree, &diffopts)) < 0) return; + + size_t n = git_diff_num_deltas(diff); + fprintf(stderr, "[pre-checkout] deltas for %s: %zu\n", path_rel, n); + git_diff_free(diff); diff = NULL; + + if (n == 0) { + fprintf(stderr, "No changes to apply for %s (already matches target or not selected)\n", path_rel); + /* fall through: we can still attempt checkout to let planner confirm */ + } + + /* 4) Configure checkout for a single literal path and creation allowed */ + const char *paths[] = { path_rel }; + opts.checkout_strategy = GIT_CHECKOUT_FORCE | GIT_CHECKOUT_SAFE | GIT_CHECKOUT_DISABLE_PATHSPEC_MATCH; // or GIT_CHECKOUT_FORCE to overwrite local edits + opts.paths.strings = (char **)paths; + opts.paths.count = 1; + opts.notify_flags = GIT_CHECKOUT_NOTIFY_ALL; + opts.notify_cb = on_notify; + + /* Optional: ensure baseline reflects current HEAD */ + // git_object *head = NULL; git_tree *head_tree = NULL; + // if (git_revparse_single(&head, repo, "HEAD^{commit}") == 0) { + // git_commit_tree(&head_tree, (git_commit *)head); + // opts.baseline = head_tree; + // } + + /* 5) Only the selected path will be considered; planner will create/update it */ + err = git_checkout_tree(repo, (git_object *)tree, &opts); + + git_tree_free(tree); + git_commit_free(commit); + git_object_free(commitobj); + return; +} + Status HfDownloader::downloadModel() { if (FileSystem::isPathEscaped(this->downloadPath)) { @@ -827,7 +901,15 @@ Status HfDownloader::downloadModel() { for (const auto& p : matches) { std::cout << " Resuming " << p.string() << "\n"; - resumeLfsDownloadForFile(repo, p, this->downloadPath); + // Remove existing lfs pointer file from repository + std::string fullPath = FileSystem::joinPath({this->downloadPath, p.string()}); + std::filesystem::path filePath(fullPath); + if (!std::filesystem::remove(filePath)) { + SPDLOG_ERROR("Removing lfs file pointer error {}", fullPath); + return StatusCode::HF_GIT_CLONE_FAILED; + } + std::string path = p.string(); + resumeLfsDownloadForFile1(repo, path.c_str()); } // Use proxy From 50b2fcf393189059cb2f1df1d597429489764e98 Mon Sep 17 00:00:00 2001 From: rasapala Date: Tue, 3 Mar 2026 16:14:43 +0100 Subject: [PATCH 08/33] Add resume in libgit2 on existing file --- third_party/libgit2/lfs.patch | 84 ++++++++++++++++++++--------------- 1 file changed, 47 insertions(+), 37 deletions(-) diff --git a/third_party/libgit2/lfs.patch b/third_party/libgit2/lfs.patch index 3b2e8a5024..c70f0bbe37 100644 --- a/third_party/libgit2/lfs.patch +++ b/third_party/libgit2/lfs.patch @@ -312,10 +312,10 @@ index 58cb4b424..00ddee9f3 100644 git_writestream **out, diff --git a/src/libgit2/lfs_filter.c b/src/libgit2/lfs_filter.c new file mode 100644 -index 000000000..7c1111c34 +index 000000000..9551461e7 --- /dev/null +++ b/src/libgit2/lfs_filter.c -@@ -0,0 +1,655 @@ +@@ -0,0 +1,665 @@ +/* +/ Copyright 2025 Intel Corporation +/ @@ -715,6 +715,38 @@ index 000000000..7c1111c34 + status = setopt; \ + } + ++int get_curl_resume_url(CURL *dl_curl, struct FtpFile* ftpfile) ++{ ++ curl_off_t resume_from = 0; ++ curl_easy_getinfo( ++ dl_curl, CURLINFO_CONTENT_LENGTH_DOWNLOAD_T, &resume_from); ++ ++ if (resume_from == -1) { ++ fprintf(stderr, ++ "\n[ERROR] curl_easy_perform() failed with transferred a partial file error and server does not support range/resume.\n"); ++ } else { ++ printf("\n[INFO] curl_easy_perform() trying to resume file download\n"); ++ curl_off_t offset = 0; ++ if (ftpfile->stream) { ++ fseek(ftpfile->stream, 0, SEEK_END); ++ offset = ftell(ftpfile->stream); ++ } else { ++ ftpfile->stream = fopen(ftpfile->filename, "ab+"); ++ if (ftpfile->stream) { ++ fseek(ftpfile->stream, 0, SEEK_END); ++ offset = ftell(ftpfile->stream); ++ } ++ } ++ ++ /* Tell libcurl to resume */ ++ curl_easy_setopt(dl_curl, CURLOPT_RESUME_FROM_LARGE, offset); ++ /* Perform the request, res gets the return code */ ++ return curl_easy_perform(dl_curl); ++ } ++ ++ return resume_from; ++} ++ +/** + * lfs_download - Downloads a file using the LFS (Large File Storage) mechanism. + * @@ -868,43 +900,21 @@ index 000000000..7c1111c34 + curl_easy_cleanup(dl_curl); + goto on_error; + } -+ print_download_info(la->full_path, get_digit(la->lfs_size)); -+ /* Perform the request, res gets the return code */ -+ res = curl_easy_perform(dl_curl); ++ ++ /* Check for resume if previous download failed and we have the partial file on disk */ ++ if (fopen(ftpfile.filename, "r") != NULL) { ++ fclose(ftpfile.filename); ++ res = get_curl_resume_url(dl_curl, &ftpfile); ++ } else { ++ print_download_info( ++ la->full_path, get_digit(la->lfs_size)); ++ /* Perform the request, res gets the return code */ ++ res = curl_easy_perform(dl_curl); ++ } ++ + /* Check for resume of partial download error */ + if (res == CURLE_PARTIAL_FILE) { -+ curl_off_t resume_from = 0; -+ curl_easy_getinfo( -+ dl_curl, CURLINFO_CONTENT_LENGTH_DOWNLOAD_T, -+ &resume_from); -+ -+ if (resume_from == -1) { -+ fprintf(stderr, -+ "\n[ERROR] curl_easy_perform() failed with transferred a partial file error and server does not support range/resume.\n"); -+ } else { -+ fprintf(stderr, -+ "\n[ERROR] curl_easy_perform() failed with transferred a partial file error and trying to resume\n"); -+ curl_off_t offset = 0; -+ if (ftpfile.stream) { -+ fseek(ftpfile.stream, 0, SEEK_END); -+ offset = ftell(ftpfile.stream); -+ } else { -+ ftpfile.stream = -+ fopen(ftpfile.filename, "ab+"); -+ if (ftpfile.stream) { -+ fseek(ftpfile.stream, 0, -+ SEEK_END); -+ offset = ftell(ftpfile.stream); -+ } -+ } -+ -+ /* Tell libcurl to resume */ -+ curl_easy_setopt( -+ dl_curl, CURLOPT_RESUME_FROM_LARGE, -+ offset); -+ /* Perform the request, res gets the return code */ -+ res = curl_easy_perform(dl_curl); -+ } ++ res = get_curl_resume_url(dl_curl, &ftpfile); + } + + /* Check for errors */ From fdc0309d5a5493e9251dd92f37f41a97c972535d Mon Sep 17 00:00:00 2001 From: rasapala Date: Tue, 3 Mar 2026 16:33:26 +0100 Subject: [PATCH 09/33] Fix segfault --- third_party/libgit2/lfs.patch | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/third_party/libgit2/lfs.patch b/third_party/libgit2/lfs.patch index c70f0bbe37..7b542c4563 100644 --- a/third_party/libgit2/lfs.patch +++ b/third_party/libgit2/lfs.patch @@ -312,10 +312,10 @@ index 58cb4b424..00ddee9f3 100644 git_writestream **out, diff --git a/src/libgit2/lfs_filter.c b/src/libgit2/lfs_filter.c new file mode 100644 -index 000000000..9551461e7 +index 000000000..f74666a10 --- /dev/null +++ b/src/libgit2/lfs_filter.c -@@ -0,0 +1,665 @@ +@@ -0,0 +1,666 @@ +/* +/ Copyright 2025 Intel Corporation +/ @@ -952,9 +952,10 @@ index 000000000..9551461e7 + return; + + on_error: ++ fprintf(stderr, "\n[ERROR] LFS download failed for file %s\n", ++ la->full_path); + free(tmp_out_file); + git__free(payload); -+ fprintf(stderr, "\n[ERROR] LFS download failed for file %s\n", la->full_path); + return; +} + From 58f07d0a4118a6b826b10be06592d7fa78ee5601 Mon Sep 17 00:00:00 2001 From: Rafal Sapala Date: Tue, 3 Mar 2026 17:03:29 +0100 Subject: [PATCH 10/33] Find way to set the url --- src/pull_module/libgit2.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/pull_module/libgit2.cpp b/src/pull_module/libgit2.cpp index 8b3944e846..6953abc5c1 100644 --- a/src/pull_module/libgit2.cpp +++ b/src/pull_module/libgit2.cpp @@ -899,6 +899,11 @@ Status HfDownloader::downloadModel() { return StatusCode::HF_GIT_CLONE_FAILED; } + // Set repository url + std::string passRepoUrl = GetRepositoryUrlWithPassword(); + const char* url = passRepoUrl.c_str(); + repo->url = url; + for (const auto& p : matches) { std::cout << " Resuming " << p.string() << "\n"; // Remove existing lfs pointer file from repository From acc90984f48590b38730eabd92777b6594a7ecfe Mon Sep 17 00:00:00 2001 From: rasapala Date: Wed, 4 Mar 2026 09:35:03 +0100 Subject: [PATCH 11/33] Set repositroy url --- third_party/libgit2/lfs.patch | 38 ++++++++++++++++++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/third_party/libgit2/lfs.patch b/third_party/libgit2/lfs.patch index 7b542c4563..52c57b0480 100644 --- a/third_party/libgit2/lfs.patch +++ b/third_party/libgit2/lfs.patch @@ -24,6 +24,26 @@ index 31da49a88..d61c9735e 100644 if(BUILD_EXAMPLES) add_subdirectory(examples) endif() +diff --git a/include/git2/repository.h b/include/git2/repository.h +index b203576af..26309dd3f 100644 +--- a/include/git2/repository.h ++++ b/include/git2/repository.h +@@ -184,6 +184,15 @@ GIT_EXTERN(int) git_repository_open_ext( + unsigned int flags, + const char *ceiling_dirs); + ++/** ++ * Set repository url member ++ * ++ * ++ * @param repo repository handle to update. If NULL nothing occurs. ++ * @param url the remote repository to clone or run checkout against. ++ */ ++GIT_EXTERN(int) git_repository_set_url(git_repository *repo, const char *url); ++ + /** + * Open a bare repository on the serverside. + * diff --git a/include/git2/sys/filter.h b/include/git2/sys/filter.h index 60466d173..a35ad5f98 100644 --- a/include/git2/sys/filter.h @@ -983,7 +1003,7 @@ index 000000000..f74666a10 + return f; +} diff --git a/src/libgit2/repository.c b/src/libgit2/repository.c -index 73876424a..6c267bc98 100644 +index 73876424a..6c9bd0b75 100644 --- a/src/libgit2/repository.c +++ b/src/libgit2/repository.c @@ -190,6 +190,7 @@ void git_repository_free(git_repository *repo) @@ -994,6 +1014,22 @@ index 73876424a..6c267bc98 100644 git__memzero(repo, sizeof(*repo)); git__free(repo); +@@ -1104,6 +1105,15 @@ static int repo_is_worktree(unsigned *out, const git_repository *repo) + return error; + } + ++int git_repository_set_url( ++ git_repository *repo, ++ const char *url) ++{ ++ GIT_ASSERT_ARG(repo); ++ GIT_ASSERT_ARG(url); ++ repo->url = git__strdup(url); ++} ++ + int git_repository_open_ext( + git_repository **repo_ptr, + const char *start_path, diff --git a/src/libgit2/repository.h b/src/libgit2/repository.h index fbf143894..1890c61c1 100644 --- a/src/libgit2/repository.h From ee4462d4125a4c459a160fcddbe80fb7a701ca0f Mon Sep 17 00:00:00 2001 From: rasapala Date: Wed, 4 Mar 2026 09:37:29 +0100 Subject: [PATCH 12/33] Update --- third_party/libgit2/lfs.patch | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/third_party/libgit2/lfs.patch b/third_party/libgit2/lfs.patch index 52c57b0480..de948749b5 100644 --- a/third_party/libgit2/lfs.patch +++ b/third_party/libgit2/lfs.patch @@ -1003,7 +1003,7 @@ index 000000000..f74666a10 + return f; +} diff --git a/src/libgit2/repository.c b/src/libgit2/repository.c -index 73876424a..6c9bd0b75 100644 +index 73876424a..f374d7f51 100644 --- a/src/libgit2/repository.c +++ b/src/libgit2/repository.c @@ -190,6 +190,7 @@ void git_repository_free(git_repository *repo) @@ -1014,7 +1014,7 @@ index 73876424a..6c9bd0b75 100644 git__memzero(repo, sizeof(*repo)); git__free(repo); -@@ -1104,6 +1105,15 @@ static int repo_is_worktree(unsigned *out, const git_repository *repo) +@@ -1104,6 +1105,16 @@ static int repo_is_worktree(unsigned *out, const git_repository *repo) return error; } @@ -1025,6 +1025,7 @@ index 73876424a..6c9bd0b75 100644 + GIT_ASSERT_ARG(repo); + GIT_ASSERT_ARG(url); + repo->url = git__strdup(url); ++ return 0; +} + int git_repository_open_ext( From 553458ea31e13e81970e2c377560df6bb19b9794 Mon Sep 17 00:00:00 2001 From: rasapala Date: Wed, 4 Mar 2026 09:48:00 +0100 Subject: [PATCH 13/33] Fix file --- third_party/libgit2/lfs.patch | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/third_party/libgit2/lfs.patch b/third_party/libgit2/lfs.patch index de948749b5..79159dcf1e 100644 --- a/third_party/libgit2/lfs.patch +++ b/third_party/libgit2/lfs.patch @@ -332,10 +332,10 @@ index 58cb4b424..00ddee9f3 100644 git_writestream **out, diff --git a/src/libgit2/lfs_filter.c b/src/libgit2/lfs_filter.c new file mode 100644 -index 000000000..f74666a10 +index 000000000..cc540d0c7 --- /dev/null +++ b/src/libgit2/lfs_filter.c -@@ -0,0 +1,666 @@ +@@ -0,0 +1,669 @@ +/* +/ Copyright 2025 Intel Corporation +/ @@ -922,8 +922,10 @@ index 000000000..f74666a10 + } + + /* Check for resume if previous download failed and we have the partial file on disk */ -+ if (fopen(ftpfile.filename, "r") != NULL) { -+ fclose(ftpfile.filename); ++ ftpfile.stream = fopen(ftpfile.filename, "r"); ++ if (ftpfile.stream != NULL) ++ { ++ fclose(ftpfile.stream); + res = get_curl_resume_url(dl_curl, &ftpfile); + } else { + print_download_info( @@ -974,6 +976,7 @@ index 000000000..f74666a10 + on_error: + fprintf(stderr, "\n[ERROR] LFS download failed for file %s\n", + la->full_path); ++ fflush(stderr); + free(tmp_out_file); + git__free(payload); + return; From e9b08a7bbf7a246c32e4d758ae5aec6ddf9e8db9 Mon Sep 17 00:00:00 2001 From: rasapala Date: Wed, 4 Mar 2026 10:29:17 +0100 Subject: [PATCH 14/33] Force resume --- third_party/libgit2/lfs.patch | 55 ++++++++++++++++++++++------------- 1 file changed, 34 insertions(+), 21 deletions(-) diff --git a/third_party/libgit2/lfs.patch b/third_party/libgit2/lfs.patch index 79159dcf1e..c4b6723d4b 100644 --- a/third_party/libgit2/lfs.patch +++ b/third_party/libgit2/lfs.patch @@ -332,10 +332,10 @@ index 58cb4b424..00ddee9f3 100644 git_writestream **out, diff --git a/src/libgit2/lfs_filter.c b/src/libgit2/lfs_filter.c new file mode 100644 -index 000000000..cc540d0c7 +index 000000000..f76f80a73 --- /dev/null +++ b/src/libgit2/lfs_filter.c -@@ -0,0 +1,669 @@ +@@ -0,0 +1,682 @@ +/* +/ Copyright 2025 Intel Corporation +/ @@ -737,6 +737,7 @@ index 000000000..cc540d0c7 + +int get_curl_resume_url(CURL *dl_curl, struct FtpFile* ftpfile) +{ ++ /* + curl_off_t resume_from = 0; + curl_easy_getinfo( + dl_curl, CURLINFO_CONTENT_LENGTH_DOWNLOAD_T, &resume_from); @@ -745,26 +746,24 @@ index 000000000..cc540d0c7 + fprintf(stderr, + "\n[ERROR] curl_easy_perform() failed with transferred a partial file error and server does not support range/resume.\n"); + } else { -+ printf("\n[INFO] curl_easy_perform() trying to resume file download\n"); -+ curl_off_t offset = 0; ++ */ ++ printf("\n[INFO] curl_easy_perform() trying to resume file download\n"); ++ curl_off_t offset = 0; ++ if (ftpfile->stream) { ++ fseek(ftpfile->stream, 0, SEEK_END); ++ offset = ftell(ftpfile->stream); ++ } else { ++ ftpfile->stream = fopen(ftpfile->filename, "ab+"); + if (ftpfile->stream) { + fseek(ftpfile->stream, 0, SEEK_END); + offset = ftell(ftpfile->stream); -+ } else { -+ ftpfile->stream = fopen(ftpfile->filename, "ab+"); -+ if (ftpfile->stream) { -+ fseek(ftpfile->stream, 0, SEEK_END); -+ offset = ftell(ftpfile->stream); -+ } + } -+ -+ /* Tell libcurl to resume */ -+ curl_easy_setopt(dl_curl, CURLOPT_RESUME_FROM_LARGE, offset); -+ /* Perform the request, res gets the return code */ -+ return curl_easy_perform(dl_curl); + } + -+ return resume_from; ++ /* Tell libcurl to resume */ ++ curl_easy_setopt(dl_curl, CURLOPT_RESUME_FROM_LARGE, offset); ++ /* Perform the request, res gets the return code */ ++ return curl_easy_perform(dl_curl); +} + +/** @@ -921,11 +920,24 @@ index 000000000..cc540d0c7 + goto on_error; + } + ++ curl_off_t resume_from = 0; ++ curl_easy_getinfo( ++ dl_curl, CURLINFO_CONTENT_LENGTH_DOWNLOAD_T, ++ &resume_from); ++ ++ if (resume_from == -1) { ++ fprintf(stderr, ++ "\n[ERROR] curl_easy_perform() failed with transferred a partial file error and server does not support range/resume.\n"); ++ } else { ++ printf("\n[INFO] curl_easy_perform() trying to resume file download\n"); ++ } ++ + /* Check for resume if previous download failed and we have the partial file on disk */ + ftpfile.stream = fopen(ftpfile.filename, "r"); + if (ftpfile.stream != NULL) + { + fclose(ftpfile.stream); ++ ftpfile.stream = NULL; + res = get_curl_resume_url(dl_curl, &ftpfile); + } else { + print_download_info( @@ -943,18 +955,19 @@ index 000000000..cc540d0c7 + if (res != CURLE_OK) { + fprintf(stderr, "\n[ERROR] curl_easy_perform() failed: %s\n", + curl_easy_strerror(res)); -+ if (ftpfile.stream) ++ if (ftpfile.stream) { + fclose(ftpfile.stream); ++ ftpfile.stream = NULL; ++ } + /* always cleanup */ + curl_easy_cleanup(dl_curl); -+ -+ /* Add partial file status */ -+ + goto on_error; + } + -+ if (ftpfile.stream) ++ if (ftpfile.stream) { + fclose(ftpfile.stream); ++ ftpfile.stream = NULL; ++ } + /* always cleanup */ + curl_easy_cleanup(dl_curl); + } From 10780320d36434e19a45edcb1bb63aae0dfcffaa Mon Sep 17 00:00:00 2001 From: Rafal Sapala Date: Thu, 5 Mar 2026 12:46:51 +0100 Subject: [PATCH 15/33] Unit tests for resume --- src/pull_module/libgit2.cpp | 12 ++++- src/test/pull_hf_model_test.cpp | 80 +++++++++++++++++++++++++++++++++ 2 files changed, 91 insertions(+), 1 deletion(-) diff --git a/src/pull_module/libgit2.cpp b/src/pull_module/libgit2.cpp index 6953abc5c1..0523c06df0 100644 --- a/src/pull_module/libgit2.cpp +++ b/src/pull_module/libgit2.cpp @@ -902,7 +902,17 @@ Status HfDownloader::downloadModel() { // Set repository url std::string passRepoUrl = GetRepositoryUrlWithPassword(); const char* url = passRepoUrl.c_str(); - repo->url = url; + error = git_repository_set_url(repo, url); + if (error < 0) { + const git_error *err = git_error_last(); + if (err) + SPDLOG_ERROR("Repository set url failed: {} {}", err->klass, err->message); + else + SPDLOG_ERROR("Repository set url failed: {}", error); + if (repo) git_repository_free(repo); + std::cout << "Path already exists on local filesystem. And set git repository url failed: " << this->downloadPath << std::endl; + return StatusCode::HF_GIT_CLONE_FAILED; + } for (const auto& p : matches) { std::cout << " Resuming " << p.string() << "\n"; diff --git a/src/test/pull_hf_model_test.cpp b/src/test/pull_hf_model_test.cpp index b29bbee326..d5b368dba9 100644 --- a/src/test/pull_hf_model_test.cpp +++ b/src/test/pull_hf_model_test.cpp @@ -39,6 +39,8 @@ #include "environment.hpp" +namespace fs = std::filesystem; + class HfDownloaderPullHfModel : public TestWithTempDir { protected: ovms::Server& server = ovms::Server::instance(); @@ -168,6 +170,84 @@ TEST_F(HfDownloaderPullHfModel, PositiveDownload) { ASSERT_EQ(expectedGraphContents, removeVersionString(graphContents)) << graphContents; } +// Truncate the file to half its size, keeping the first half. +bool removeSecondHalf(const std::string& filrStr) { + const fs::path& file(filrStr); + std::error_code ec; + ec.clear(); + + if (!fs::exists(file, ec) || !fs::is_regular_file(file, ec)) { + if (!ec) ec = std::make_error_code(std::errc::no_such_file_or_directory); + return false; + } + + const std::uintmax_t size = fs::file_size(file, ec); + if (ec) return false; + + const std::uintmax_t newSize = size / 2; // floor(size/2) + fs::resize_file(file, newSize, ec); + return !ec; +} + +bool createGitLfsPointerFile(const std::string& path) { + std::ofstream file(path, std::ios::binary); + if (!file.is_open()) { + return false; + } + + file << + "version https://git-lfs.github.com/spec/v1\n" + "oid sha256:59f24bc922e1a48bb3feeba18b23f0e9622a7ee07166d925650d7a933283f8b1\n" + "size 123882252\n"; + + return true; +} + +TEST_F(HfDownloaderPullHfModel, PositiveDownloadAndResumeFromPArtialDownload) { + std::string modelName = "OpenVINO/Phi-3-mini-FastDraft-50M-int8-ov"; + std::string downloadPath = ovms::FileSystem::joinPath({this->directoryPath, "repository"}); + std::string task = "text_generation"; + this->ServerPullHfModel(modelName, downloadPath, task); + server.setShutdownRequest(1); + if (t) + t->join(); + server.setShutdownRequest(0); + + std::string ovModelName = "openvino_model.bin"; + std::string basePath = ovms::FileSystem::joinPath({this->directoryPath, "repository", "OpenVINO", "Phi-3-mini-FastDraft-50M-int8-ov"}); + std::string modelPath = ovms::FileSystem::appendSlash(basePath) + ovModelName; + std::string graphPath = ovms::FileSystem::appendSlash(basePath) + "graph.pbtxt"; + + ASSERT_EQ(std::filesystem::exists(modelPath), true) << modelPath; + ASSERT_EQ(std::filesystem::exists(graphPath), true) << graphPath; + ASSERT_EQ(std::filesystem::file_size(modelPath), 52417240); + std::string graphContents = GetFileContents(graphPath); + + ASSERT_EQ(expectedGraphContents, removeVersionString(graphContents)) << graphContents; + + // Prepare a git repository with a lfs_part file and lfs pointer file to simulate partial download error of a big model + ASSERT_EQ(removeSecondHalf(modelPath), true); + ASSERT_EQ(std::filesystem::file_size(modelPath), 26208620); + std::error_code ec; + ec.clear(); + std::string ovModelPartLfsName = "openvino_model.binlfs_part"; + std::string ovModelPartLfsPath = ovms::FileSystem::appendSlash(basePath) + ovModelPartLfsName; + fs::rename(modelPath, ovModelPartLfsPath, ec); + ASSERT_EQ(ec, std::errc()); + ASSERT_EQ(std::filesystem::file_size(ovModelPartLfsPath), 26208620); + ASSERT_EQ(createGitLfsPointerFile(modelPath), true); + + // Call ovms pull to resume the file + this->ServerPullHfModel(modelName, downloadPath, task); + + ASSERT_EQ(std::filesystem::exists(modelPath), true) << modelPath; + ASSERT_EQ(std::filesystem::exists(graphPath), true) << graphPath; + ASSERT_EQ(std::filesystem::file_size(modelPath), 52417240); + graphContents = GetFileContents(graphPath); + + ASSERT_EQ(expectedGraphContents, removeVersionString(graphContents)) << graphContents; +} + TEST_F(HfDownloaderPullHfModel, PositiveDownloadAndStart) { SKIP_AND_EXIT_IF_NOT_RUNNING_UNSTABLE(); // CVS-180127 // EnvGuard guard; From 7d4498198238cf667fda921eb11156b28595b4d0 Mon Sep 17 00:00:00 2001 From: rasapala Date: Thu, 5 Mar 2026 12:47:29 +0100 Subject: [PATCH 16/33] Fix rename --- third_party/libgit2/lfs.patch | 31 +++++++++++++------------------ 1 file changed, 13 insertions(+), 18 deletions(-) diff --git a/third_party/libgit2/lfs.patch b/third_party/libgit2/lfs.patch index c4b6723d4b..0197a68564 100644 --- a/third_party/libgit2/lfs.patch +++ b/third_party/libgit2/lfs.patch @@ -332,10 +332,10 @@ index 58cb4b424..00ddee9f3 100644 git_writestream **out, diff --git a/src/libgit2/lfs_filter.c b/src/libgit2/lfs_filter.c new file mode 100644 -index 000000000..f76f80a73 +index 000000000..f877c4afa --- /dev/null +++ b/src/libgit2/lfs_filter.c -@@ -0,0 +1,682 @@ +@@ -0,0 +1,677 @@ +/* +/ Copyright 2025 Intel Corporation +/ @@ -887,6 +887,7 @@ index 000000000..f76f80a73 + } + + /* get a curl handle */ ++ bool resumingFileByBlobFilter = false; + dl_curl = curl_easy_init(); + if (dl_curl) { + struct FtpFile ftpfile = { tmp_out_file, NULL }; @@ -920,22 +921,11 @@ index 000000000..f76f80a73 + goto on_error; + } + -+ curl_off_t resume_from = 0; -+ curl_easy_getinfo( -+ dl_curl, CURLINFO_CONTENT_LENGTH_DOWNLOAD_T, -+ &resume_from); -+ -+ if (resume_from == -1) { -+ fprintf(stderr, -+ "\n[ERROR] curl_easy_perform() failed with transferred a partial file error and server does not support range/resume.\n"); -+ } else { -+ printf("\n[INFO] curl_easy_perform() trying to resume file download\n"); -+ } -+ + /* Check for resume if previous download failed and we have the partial file on disk */ + ftpfile.stream = fopen(ftpfile.filename, "r"); + if (ftpfile.stream != NULL) + { ++ resumingFileByBlobFilter = true; + fclose(ftpfile.stream); + ftpfile.stream = NULL; + res = get_curl_resume_url(dl_curl, &ftpfile); @@ -972,10 +962,15 @@ index 000000000..f76f80a73 + curl_easy_cleanup(dl_curl); + } + -+ /* Remove lfs file and rename downloaded file to oryginal lfs filename */ -+ if (p_unlink(la->full_path) < 0) { -+ fprintf(stderr, "\n[ERROR] failed to delete file '%s'\n", la->full_path); -+ goto on_error; ++ /* Remove lfs file and rename downloaded file to oryginal lfs filename */ ++ if (!resumingFileByBlobFilter) { ++ /* File does not exist when using blob filters */ ++ if (p_unlink(la->full_path) < 0) { ++ fprintf(stderr, ++ "\n[ERROR] failed to delete file '%s'\n", ++ la->full_path); ++ goto on_error; ++ } + } + + if (p_rename(tmp_out_file, la->full_path) < 0) { From 5aae45443410f167f7896d10f604a828701a1f47 Mon Sep 17 00:00:00 2001 From: Rafal Sapala Date: Mon, 9 Mar 2026 14:59:13 +0100 Subject: [PATCH 17/33] Unit test --- src/pull_module/libgit2.cpp | 485 ++------------------------------ src/pull_module/libgit2.hpp | 2 +- src/test/pull_hf_model_test.cpp | 67 ++++- 3 files changed, 80 insertions(+), 474 deletions(-) diff --git a/src/pull_module/libgit2.cpp b/src/pull_module/libgit2.cpp index 0523c06df0..570ecd08fe 100644 --- a/src/pull_module/libgit2.cpp +++ b/src/pull_module/libgit2.cpp @@ -180,7 +180,7 @@ Status HfDownloader::RemoveReadonlyFileAttributeFromDir(const std::string& direc return StatusCode::OK; } -Status HfDownloader::CheckRepositoryStatus() { +Status HfDownloader::CheckRepositoryStatus(bool checkUntracked) { git_repository *repo = NULL; int error = git_repository_open_ext(&repo, this->downloadPath.c_str(), 0, NULL); if (error < 0) { @@ -253,84 +253,13 @@ Status HfDownloader::CheckRepositoryStatus() { SPDLOG_DEBUG(ss.str()); git_status_list_free(status_list); - if (is_unborn || is_detached || staged || unstaged || untracked || conflicted) { + // We do not care about untracked until after git clone + if (is_unborn || is_detached || staged || unstaged || conflicted || (checkUntracked && untracked)) { return StatusCode::HF_GIT_STATUS_UNCLEAN; } return StatusCode::OK; } -static int print_changed_and_untracked(git_repository *repo) { - int error = 0; - git_status_list *statuslist = NULL; - - git_status_options opts; - error = git_status_options_init(&opts, GIT_STATUS_OPTIONS_VERSION); - if (error < 0) return error; - - // Choose what to include - opts.show = GIT_STATUS_SHOW_INDEX_AND_WORKDIR; // consider both index and working dir - opts.flags = - GIT_STATUS_OPT_INCLUDE_UNTRACKED | // include untracked files - GIT_STATUS_OPT_RECURSE_UNTRACKED_DIRS | // recurse into untracked dirs - GIT_STATUS_OPT_INCLUDE_IGNORED | // (optional) include ignored if you want to see them - GIT_STATUS_OPT_RENAMES_HEAD_TO_INDEX | // detect renames in index - GIT_STATUS_OPT_RENAMES_INDEX_TO_WORKDIR | // detect renames in workdir - GIT_STATUS_OPT_SORT_CASE_SENSITIVELY; // stable ordering - - // If you want to limit to certain paths/patterns, set opts.pathspec here. - - if ((error = git_status_list_new(&statuslist, repo, &opts)) < 0) - return error; - - size_t count = git_status_list_entrycount(statuslist); - for (size_t i = 0; i < count; i++) { - const git_status_entry *e = git_status_byindex(statuslist, i); - if (!e) continue; - - unsigned int s = e->status; - - // Consider “changed” as anything that’s not current in HEAD/INDEX/WT: - // You can tailor this to your exact definition. - int is_untracked = - (s & GIT_STATUS_WT_NEW) != 0; // working tree new (untracked) - int is_workdir_changed = - (s & (GIT_STATUS_WT_MODIFIED | - GIT_STATUS_WT_DELETED | - GIT_STATUS_WT_RENAMED | - GIT_STATUS_WT_TYPECHANGE)) != 0; - int is_index_changed = - (s & (GIT_STATUS_INDEX_NEW | - GIT_STATUS_INDEX_MODIFIED | - GIT_STATUS_INDEX_DELETED | - GIT_STATUS_INDEX_RENAMED | - GIT_STATUS_INDEX_TYPECHANGE)) != 0; - - if (!(is_untracked || is_workdir_changed || is_index_changed)) - continue; - - // Prefer the most relevant delta for the path - const git_diff_delta *delta = NULL; - if (is_workdir_changed && e->index_to_workdir) - delta = e->index_to_workdir; - else if (is_index_changed && e->head_to_index) - delta = e->head_to_index; - else if (is_untracked && e->index_to_workdir) - delta = e->index_to_workdir; - - if (!delta) continue; - - // For renames, old_file and new_file may differ; typically you want new_file.path - const char *path = delta->new_file.path ? delta->new_file.path - : delta->old_file.path; - - // Print or collect the filename - SPDLOG_INFO("is_untracked {} is_workdir_changed {} is_index_changed {} File {} ", is_untracked, is_workdir_changed, is_index_changed, path); - } - - git_status_list_free(statuslist); - return 0; -} - #define CHECK(call) do { \ int _err = (call); \ if (_err < 0) { \ @@ -340,210 +269,6 @@ static int print_changed_and_untracked(git_repository *repo) { } \ } while (0) -// Fetch from remote and update FETCH_HEAD -static void do_fetch(git_repository *repo, const char *remote_name, const char *proxy) -{ - git_remote *remote = NULL; - git_fetch_options fetch_opts = GIT_FETCH_OPTIONS_INIT; - - fetch_opts.prune = GIT_FETCH_PRUNE_UNSPECIFIED; - fetch_opts.update_fetchhead = 1; - fetch_opts.download_tags = GIT_REMOTE_DOWNLOAD_TAGS_ALL; - fetch_opts.callbacks = (git_remote_callbacks) GIT_REMOTE_CALLBACKS_INIT; - fetch_opts.callbacks.credentials = cred_acquire_cb; - if (proxy) { - fetch_opts.proxy_opts.type = GIT_PROXY_SPECIFIED; - fetch_opts.proxy_opts.url = proxy; - } - - CHECK(git_remote_lookup(&remote, repo, remote_name)); - - printf("Fetching from %s...\n", remote_name); - CHECK(git_remote_fetch(remote, NULL, &fetch_opts, NULL)); - - // Optional: update remote-tracking branches' default refspec tips - // (git_remote_fetch already updates tips if update_fetchhead=1; explicit - // update_tips is not required in recent libgit2 versions.) - - git_remote_free(remote); -} - -// Fast-forward the local branch to target OID -static void do_fast_forward(git_repository *repo, - git_reference *local_branch, - const git_oid *target_oid) -{ - git_object *target = NULL; - git_checkout_options co_opts = GIT_CHECKOUT_OPTIONS_INIT; - git_reference *updated_ref = NULL; - - CHECK(git_object_lookup(&target, repo, target_oid, GIT_OBJECT_COMMIT)); - - // Update the branch reference to point to the target commit - CHECK(git_reference_set_target(&updated_ref, local_branch, target_oid, "Fast-forward")); - - // Make sure HEAD points to that branch (it normally already does) - CHECK(git_repository_set_head(repo, git_reference_name(updated_ref))); - - // Checkout files to match the target tree - co_opts.checkout_strategy = GIT_CHECKOUT_SAFE; - CHECK(git_checkout_tree(repo, target, &co_opts)); - - printf("Fast-forwarded %s to %s\n", - git_reference_shorthand(local_branch), - git_oid_tostr_s(target_oid)); - - git_object_free(target); - git_reference_free(updated_ref); -} - -// Perform a normal merge and create a merge commit if no conflicts -static void do_normal_merge(git_repository *repo, - git_reference *local_branch, - git_annotated_commit *their_head) -{ - git_merge_options merge_opts = GIT_MERGE_OPTIONS_INIT; - git_checkout_options co_opts = GIT_CHECKOUT_OPTIONS_INIT; - - merge_opts.file_favor = GIT_MERGE_FILE_FAVOR_NORMAL; - co_opts.checkout_strategy = GIT_CHECKOUT_SAFE | GIT_CHECKOUT_RECREATE_MISSING; - - const git_annotated_commit *their_heads[1] = { their_head }; - - printf("Merging...\n"); - CHECK(git_merge(repo, their_heads, 1, &merge_opts, &co_opts)); - - // Check for conflicts - git_index *index = NULL; - CHECK(git_repository_index(&index, repo)); - - if (git_index_has_conflicts(index)) { - printf("Merge has conflicts. Please resolve them and create the merge commit manually.\n"); - git_index_free(index); - return; // Leave repository in merging state - } - - // Write index to tree - git_oid tree_oid; - CHECK(git_index_write_tree(&tree_oid, index)); - CHECK(git_index_write(index)); - git_index_free(index); - - git_tree *tree = NULL; - CHECK(git_tree_lookup(&tree, repo, &tree_oid)); - - // Prepare signature (from config if available) - git_signature *sig = NULL; - int err = git_signature_default(&sig, repo); - if (err == GIT_ENOTFOUND || sig == NULL) { - // Fallback if user.name/email not set in config - CHECK(git_signature_now(&sig, "Your Name", "you@example.com")); - } else { - CHECK(err); - } - - // Get current HEAD (our) commit and their commit to be parents - git_reference *head = NULL, *resolved_branch = NULL; - CHECK(git_repository_head(&head, repo)); - CHECK(git_reference_resolve(&resolved_branch, head)); // ensure direct ref - - const git_oid *our_oid = git_reference_target(resolved_branch); - git_commit *our_commit = NULL; - git_commit *their_commit = NULL; - CHECK(git_commit_lookup(&our_commit, repo, our_oid)); - CHECK(git_commit_lookup(&their_commit, repo, git_annotated_commit_id(their_head))); - - const git_commit *parents[2] = { our_commit, their_commit }; - git_oid merge_commit_oid; - - // Create merge commit on the current branch ref - CHECK(git_commit_create(&merge_commit_oid, - repo, - git_reference_name(resolved_branch), - sig, sig, - NULL /* message_encoding */, - "Merge remote-tracking branch", - tree, - 2, parents)); - - printf("Created merge commit %s on %s\n", - git_oid_tostr_s(&merge_commit_oid), - git_reference_shorthand(resolved_branch)); - - // Cleanup - git_signature_free(sig); - git_tree_free(tree); - git_commit_free(our_commit); - git_commit_free(their_commit); - git_reference_free(head); - git_reference_free(resolved_branch); -} - -// Main pull routine: fetch + merge (fast-forward if possible) -static void pull(git_repository *repo, const char *remote_name, const char *proxy) -{ - // Ensure we are on a branch (not detached HEAD) - git_reference *head = NULL; - int head_res = git_repository_head(&head, repo); - // HEAD state info - bool is_detached = git_repository_head_detached(repo) == 1; - bool is_unborn = git_repository_head_unborn(repo) == 1; - if (is_unborn) { - fprintf(stderr, "Repository has no HEAD yet (unborn branch).\n"); - return; - } else if (is_detached) { - fprintf(stderr, "HEAD is detached; cannot pull safely. Checkout a branch first.\n"); - return; - } - CHECK(head_res); - - // Resolve symbolic HEAD to direct branch ref (refs/heads/…) - git_reference *local_branch = NULL; - CHECK(git_reference_resolve(&local_branch, head)); - - // Find the upstream tracking branch (refs/remotes//) - git_reference *upstream = NULL; - int up_ok = git_branch_upstream(&upstream, local_branch); - if (up_ok != 0 || upstream == NULL) { - fprintf(stderr, "Current branch has no upstream. Set it with:\n" - " git branch --set-upstream-to=%s/ \n", remote_name); - return; - } - - // Verify upstream belongs to the requested remote; not strictly required for fetch - // but we fetch from the chosen remote anyway. - do_fetch(repo, remote_name, proxy); - - // Prepare "their" commit as annotated commit from upstream - git_annotated_commit *their_head = NULL; - CHECK(git_annotated_commit_from_ref(&their_head, repo, upstream)); - - // Merge analysis - git_merge_analysis_t analysis; - git_merge_preference_t preference; - CHECK(git_merge_analysis(&analysis, &preference, repo, - (const git_annotated_commit **)&their_head, 1)); - - if (analysis & GIT_MERGE_ANALYSIS_UP_TO_DATE) { - printf("Already up to date.\n"); - } else if (analysis & GIT_MERGE_ANALYSIS_FASTFORWARD) { - const git_oid *target_oid = git_annotated_commit_id(their_head); - do_fast_forward(repo, local_branch, target_oid); - } else if (analysis & GIT_MERGE_ANALYSIS_NORMAL) { - do_normal_merge(repo, local_branch, their_head); - } else { - printf("No merge action taken (analysis=%u, preference=%u).\n", - (unsigned)analysis, (unsigned)preference); - } - - // Cleanup - git_annotated_commit_free(their_head); - git_reference_free(upstream); - git_reference_free(local_branch); - git_reference_free(head); -} - - // Trim trailing '\r' (for CRLF files) and surrounding spaces static inline void rtrimCrLfWhitespace(std::string& s) { if (!s.empty() && s.back() == '\r') s.pop_back(); // remove trailing '\r' @@ -664,96 +389,7 @@ std::vector findLfsLikeFiles(const std::string& directory, bool recurs return matches; } -int HfDownloader::CheckRepositoryForResume() { - git_repository *repo = NULL; - int error = git_repository_open_ext(&repo, this->downloadPath.c_str(), 0, NULL); - if (error < 0) { - const git_error *err = git_error_last(); - if (err) - SPDLOG_ERROR("Repository open failed: {} {}", err->klass, err->message); - else - SPDLOG_ERROR("Repository open failed: {}", error); - if (repo) git_repository_free(repo); - - return error; - } - - error = print_changed_and_untracked(repo); - if (error < 0) { - const git_error *err = git_error_last(); - if (err) - SPDLOG_ERROR("Print changed files failed: {} {}", err->klass, err->message); - else - SPDLOG_ERROR("Print changed files failed: {}", error); - } - - if (repo) git_repository_free(repo); - return error; -} - - -/* - * checkout_one_file: Check out a single path from a treeish (commit/ref) - * into the working directory, applying filters (smudge, EOL) just like clone. - * - * repo_path : filesystem path to the existing (non-bare) repository - * treeish : e.g., "HEAD", "origin/main", a full commit SHA, etc. - * path_in_repo : repo-relative path (e.g., "src/main.c") - * - * Returns 0 on success; <0 (libgit2 error code) on failure. - */ -void resumeLfsDownloadForFile2(git_repository *repo, fs::path fileToResume, const std::string& repositoryPath) { - int error = 0; - - // Remove existing lfs pointer file from repository - std::string fullPath = FileSystem::joinPath({repositoryPath, fileToResume.string()}); - std::filesystem::path filePath(fullPath); - if (!std::filesystem::remove(filePath)) { - SPDLOG_ERROR("Removing lfs file pointer error {}", fullPath); - return; - } - - const char *path_in_repo = fileToResume.string().c_str(); - // TODO: make sure we are on 'origin/main'. - const char *treeish = "origin/main^{tree}"; - git_object *target = NULL; - git_strarray paths = {0}; - git_checkout_options opts = GIT_CHECKOUT_OPTIONS_INIT; - opts.disable_filters = 0; // default; ensure you are NOT disabling filters - - if (git_repository_is_bare(repo)) { - SPDLOG_ERROR("Repository is bare; cannot checkout to working directory {}", fileToResume.string()); - error = GIT_EBAREREPO; - goto done; - } - - if ((error = git_revparse_single(&target, repo, treeish)) < 0) { - SPDLOG_ERROR("git_revparse_single failed {}", fileToResume.string()); - goto done; - } - - // Restrict checkout to a single path - paths.count = 1; - paths.strings = (char **)&path_in_repo; - - opts.paths = paths; - - // Strategy: SAFER defaults — apply filters, write new files, update existing file - // You can add GIT_CHECKOUT_FORCE if you want to overwrite conflicts. - // opts.checkout_strategy = GIT_CHECKOUT_SAFE | GIT_CHECKOUT_DISABLE_PATHSPEC_MATCH; - opts.checkout_strategy = GIT_CHECKOUT_FORCE | GIT_CHECKOUT_SAFE; // This makes sure BLOB update with filter is called - // This actually writes the filtered content to the working directory - error = git_checkout_tree(repo, target, &opts); - if (error < 0) { - SPDLOG_ERROR("git_checkout_tree failed {}", fileToResume.string()); - } - -done: - if (target) git_object_free(target); - return; -} - -void resumeLfsDownloadForFile1(git_repository *repo, const char *file_path_in_repo) { +void resumeLfsDownloadForFile(git_repository *repo, const char *file_path_in_repo) { git_object *obj = NULL; git_tree *tree = NULL; git_tree_entry *entry = NULL; @@ -779,9 +415,6 @@ void resumeLfsDownloadForFile1(git_repository *repo, const char *file_path_in_re // Apply filters based on .gitattributes for this path CHECK(git_blob_filter(&out, blob, file_path_in_repo, &opts) != 0); - // out.ptr now contains the filtered content - fwrite(out.ptr, 1, out.size, stdout); - git_buf_dispose(&out); if (blob) git_blob_free(blob); if (entry) git_tree_entry_free(entry); @@ -790,82 +423,6 @@ void resumeLfsDownloadForFile1(git_repository *repo, const char *file_path_in_re return; } - -static int on_notify( - git_checkout_notify_t why, const char *path, - const git_diff_file *baseline, const git_diff_file *target, const git_diff_file *workdir, - void *payload) -{ - (void)baseline; (void)target; (void)workdir; (void)payload; - fprintf(stderr, "[checkout notify] why=%u path=%s\n", why, path ? path : "(null)"); - return 0; // non-zero would cancel -} - -void checkout_one_from_origin_master(git_repository *repo, const char *path_rel) { - int err = 0; - git_object *commitobj = NULL; - git_commit *commit = NULL; - git_tree *tree = NULL; - git_tree_entry *te = NULL; - git_diff *diff = NULL; - git_diff_options diffopts = GIT_DIFF_OPTIONS_INIT; - git_checkout_options opts = GIT_CHECKOUT_OPTIONS_INIT; - - - printf("Path to resume '%s' \n", path_rel); - /* 1) Resolve origin/master to a commit, then get its tree */ - if ((err = git_revparse_single(&commitobj, repo, "refs/remotes/origin/main^{commit}")) < 0) return; - commit = (git_commit *)commitobj; - if ((err = git_commit_tree(&tree, commit)) < 0) return; - - /* 2) Sanity-check: does the path exist in the target tree? */ - if ((err = git_tree_entry_bypath(&te, tree, path_rel)) == GIT_ENOTFOUND) { - fprintf(stderr, "Path '%s' not found in origin/main\n", path_rel); - err = 0; return; // nothing to do - } else if (err < 0) { - return; - } - git_tree_entry_free(te); te = NULL; - - /* 3) Diff target tree -> workdir (with index) for the one path */ - diffopts.pathspec.count = 1; - diffopts.pathspec.strings = (char **)&path_rel; - if ((err = git_diff_tree_to_workdir_with_index(&diff, repo, tree, &diffopts)) < 0) return; - - size_t n = git_diff_num_deltas(diff); - fprintf(stderr, "[pre-checkout] deltas for %s: %zu\n", path_rel, n); - git_diff_free(diff); diff = NULL; - - if (n == 0) { - fprintf(stderr, "No changes to apply for %s (already matches target or not selected)\n", path_rel); - /* fall through: we can still attempt checkout to let planner confirm */ - } - - /* 4) Configure checkout for a single literal path and creation allowed */ - const char *paths[] = { path_rel }; - opts.checkout_strategy = GIT_CHECKOUT_FORCE | GIT_CHECKOUT_SAFE | GIT_CHECKOUT_DISABLE_PATHSPEC_MATCH; // or GIT_CHECKOUT_FORCE to overwrite local edits - opts.paths.strings = (char **)paths; - opts.paths.count = 1; - opts.notify_flags = GIT_CHECKOUT_NOTIFY_ALL; - opts.notify_cb = on_notify; - - /* Optional: ensure baseline reflects current HEAD */ - // git_object *head = NULL; git_tree *head_tree = NULL; - // if (git_revparse_single(&head, repo, "HEAD^{commit}") == 0) { - // git_commit_tree(&head_tree, (git_commit *)head); - // opts.baseline = head_tree; - // } - - /* 5) Only the selected path will be considered; planner will create/update it */ - err = git_checkout_tree(repo, (git_object *)tree, &opts); - - git_tree_free(tree); - git_commit_free(commit); - git_object_free(commitobj); - return; -} - - Status HfDownloader::downloadModel() { if (FileSystem::isPathEscaped(this->downloadPath)) { SPDLOG_ERROR("Path {} escape with .. is forbidden.", this->downloadPath); @@ -874,12 +431,15 @@ Status HfDownloader::downloadModel() { // Repository exists and we do not want to overwrite if (std::filesystem::is_directory(this->downloadPath) && !this->overwriteModels) { + // Checking if the download was partially finished for any files in repository auto matches = findLfsLikeFiles(this->downloadPath, true); if (matches.empty()) { - std::cout << "No files with LFS-like keywords in the first 3 lines were found.\n"; + std::cout << "No files to resume download found.\n"; + std::cout << "Path already exists on local filesystem. Skipping download to path: " << this->downloadPath << std::endl; + return StatusCode::OK; } else { - std::cout << "Found " << matches.size() << " matching file(s):\n"; + std::cout << "Found " << matches.size() << " file(s) to resume partial download:\n"; for (const auto& p : matches) { std::cout << " " << p.string() << "\n"; } @@ -915,28 +475,17 @@ Status HfDownloader::downloadModel() { } for (const auto& p : matches) { - std::cout << " Resuming " << p.string() << "\n"; - // Remove existing lfs pointer file from repository - std::string fullPath = FileSystem::joinPath({this->downloadPath, p.string()}); - std::filesystem::path filePath(fullPath); - if (!std::filesystem::remove(filePath)) { - SPDLOG_ERROR("Removing lfs file pointer error {}", fullPath); - return StatusCode::HF_GIT_CLONE_FAILED; - } + std::cout << " Resuming " << p.string() << "...\n"; std::string path = p.string(); - resumeLfsDownloadForFile1(repo, path.c_str()); + resumeLfsDownloadForFile(repo, path.c_str()); } - - // Use proxy - if (CheckIfProxySet()) { - SPDLOG_DEBUG("Download using https_proxy settings"); - //pull(repo, "origin", this->httpProxy.c_str()); - } else { - SPDLOG_DEBUG("Download with https_proxy not set"); - pull(repo, "origin", nullptr); + + SPDLOG_DEBUG("Checking repository status."); + auto status = CheckRepositoryStatus(false); + if (!status.ok()) { + return status; } - std::cout << "Path already exists on local filesystem. Skipping download to path: " << this->downloadPath << std::endl; return StatusCode::OK; } @@ -981,7 +530,7 @@ Status HfDownloader::downloadModel() { } SPDLOG_DEBUG("Checking repository status."); - status = CheckRepositoryStatus(); + status = CheckRepositoryStatus(true); if (!status.ok()) { return status; } diff --git a/src/pull_module/libgit2.hpp b/src/pull_module/libgit2.hpp index 755a84e89e..b8dacac0e9 100644 --- a/src/pull_module/libgit2.hpp +++ b/src/pull_module/libgit2.hpp @@ -62,7 +62,7 @@ class HfDownloader : public IModelDownloader { std::string GetRepositoryUrlWithPassword(); bool CheckIfProxySet(); Status RemoveReadonlyFileAttributeFromDir(const std::string& directoryPath); - Status CheckRepositoryStatus(); + Status CheckRepositoryStatus(bool checkUntracked); int CheckRepositoryForResume(); }; } // namespace ovms diff --git a/src/test/pull_hf_model_test.cpp b/src/test/pull_hf_model_test.cpp index d5b368dba9..9eba14b103 100644 --- a/src/test/pull_hf_model_test.cpp +++ b/src/test/pull_hf_model_test.cpp @@ -14,6 +14,7 @@ // limitations under the License. //***************************************************************************** #include +#include #include #include @@ -152,7 +153,7 @@ const std::string expectedGraphContentsDraft = R"( )"; TEST_F(HfDownloaderPullHfModel, PositiveDownload) { - GTEST_SKIP() << "Skipping test in CI - PositiveDownloadAndStart has full scope testing."; + // GTEST_SKIP() << "Skipping test in CI - PositiveDownloadAndStart has full scope testing."; std::string modelName = "OpenVINO/Phi-3-mini-FastDraft-50M-int8-ov"; std::string downloadPath = ovms::FileSystem::joinPath({this->directoryPath, "repository"}); std::string task = "text_generation"; @@ -197,12 +198,60 @@ bool createGitLfsPointerFile(const std::string& path) { file << "version https://git-lfs.github.com/spec/v1\n" - "oid sha256:59f24bc922e1a48bb3feeba18b23f0e9622a7ee07166d925650d7a933283f8b1\n" - "size 123882252\n"; + "oid sha256:cecf0224201415144c00cf3a6cf3350306f9c78888d631eb590939a63722fefa\n" + "size 52417240\n"; return true; } +// Returns lowercase hex SHA-256 string on success, empty string on failure. +std::string sha256File(std::string_view path, std::error_code& ec) { + ec.clear(); + + std::ifstream ifs(std::string(path), std::ios::binary); + if (!ifs) { + ec = std::make_error_code(std::errc::no_such_file_or_directory); + return {}; + } + + SHA256_CTX ctx; + if (SHA256_Init(&ctx) != 1) { + ec = std::make_error_code(std::errc::io_error); + return {}; + } + + // Read in chunks to support large files without high memory usage. + std::vector buffer(1 << 20); // 1 MiB + while (ifs) { + ifs.read(reinterpret_cast(buffer.data()), static_cast(buffer.size())); + std::streamsize got = ifs.gcount(); + if (got > 0) { + if (SHA256_Update(&ctx, buffer.data(), static_cast(got)) != 1) { + ec = std::make_error_code(std::errc::io_error); + return {}; + } + } + } + if (!ifs.eof()) { // read failed not due to EOF + ec = std::make_error_code(std::errc::io_error); + return {}; + } + + std::array digest{}; + if (SHA256_Final(digest.data(), &ctx) != 1) { + ec = std::make_error_code(std::errc::io_error); + return {}; + } + + // Convert to lowercase hex + std::ostringstream oss; + oss << std::hex << std::setfill('0') << std::nouppercase; + for (unsigned char b : digest) { + oss << std::setw(2) << static_cast(b); + } + return oss.str(); +} + TEST_F(HfDownloaderPullHfModel, PositiveDownloadAndResumeFromPArtialDownload) { std::string modelName = "OpenVINO/Phi-3-mini-FastDraft-50M-int8-ov"; std::string downloadPath = ovms::FileSystem::joinPath({this->directoryPath, "repository"}); @@ -225,11 +274,14 @@ TEST_F(HfDownloaderPullHfModel, PositiveDownloadAndResumeFromPArtialDownload) { ASSERT_EQ(expectedGraphContents, removeVersionString(graphContents)) << graphContents; + std::error_code ec; + ec.clear(); + std::string expectedDigest = sha256File(modelPath, ec); + ASSERT_EQ(ec, std::errc()); // Prepare a git repository with a lfs_part file and lfs pointer file to simulate partial download error of a big model ASSERT_EQ(removeSecondHalf(modelPath), true); ASSERT_EQ(std::filesystem::file_size(modelPath), 26208620); - std::error_code ec; - ec.clear(); + std::string ovModelPartLfsName = "openvino_model.binlfs_part"; std::string ovModelPartLfsPath = ovms::FileSystem::appendSlash(basePath) + ovModelPartLfsName; fs::rename(modelPath, ovModelPartLfsPath, ec); @@ -240,12 +292,17 @@ TEST_F(HfDownloaderPullHfModel, PositiveDownloadAndResumeFromPArtialDownload) { // Call ovms pull to resume the file this->ServerPullHfModel(modelName, downloadPath, task); + ASSERT_EQ(std::filesystem::exists(ovModelPartLfsPath), false) << modelPath; ASSERT_EQ(std::filesystem::exists(modelPath), true) << modelPath; ASSERT_EQ(std::filesystem::exists(graphPath), true) << graphPath; ASSERT_EQ(std::filesystem::file_size(modelPath), 52417240); graphContents = GetFileContents(graphPath); ASSERT_EQ(expectedGraphContents, removeVersionString(graphContents)) << graphContents; + + std::string resumedDigest = sha256File(modelPath, ec); + ASSERT_EQ(ec, std::errc()); + ASSERT_EQ(expectedDigest, resumedDigest); } TEST_F(HfDownloaderPullHfModel, PositiveDownloadAndStart) { From 3d8323b51634d22c2b8832cfd12e904cc888161d Mon Sep 17 00:00:00 2001 From: rasapala Date: Mon, 9 Mar 2026 14:59:57 +0100 Subject: [PATCH 18/33] Pass through --- third_party/libgit2/lfs.patch | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/third_party/libgit2/lfs.patch b/third_party/libgit2/lfs.patch index 0197a68564..8d4d43217e 100644 --- a/third_party/libgit2/lfs.patch +++ b/third_party/libgit2/lfs.patch @@ -332,10 +332,10 @@ index 58cb4b424..00ddee9f3 100644 git_writestream **out, diff --git a/src/libgit2/lfs_filter.c b/src/libgit2/lfs_filter.c new file mode 100644 -index 000000000..f877c4afa +index 000000000..6dc8f9650 --- /dev/null +++ b/src/libgit2/lfs_filter.c -@@ -0,0 +1,677 @@ +@@ -0,0 +1,679 @@ +/* +/ Copyright 2025 Intel Corporation +/ @@ -488,6 +488,7 @@ index 000000000..f877c4afa + lfs_oid.ptr); + return -1; + } ++ printf("\nfrom->lfs_oid %s\n", lfs_oid.ptr); + + if (get_lfs_info_match(&lfs_size, size_regexp) < 0) { + fprintf(stderr, @@ -531,8 +532,9 @@ index 000000000..f877c4afa + + if (git_filter_source_mode(src) == GIT_FILTER_SMUDGE) + return lfs_insert_id(to, from, src, payload); -+ /*else -+ * PATH for upload lfs files not needed ++ else ++ return GIT_PASSTHROUGH; ++ /* PATH for upload lfs files not needed + return lfs_remove_id(to, from); + */ + return 0; From 356dd4ed984c9a55cc4d100a9d0daf64c73555b5 Mon Sep 17 00:00:00 2001 From: rasapala Date: Tue, 10 Mar 2026 11:40:11 +0100 Subject: [PATCH 19/33] Lfs upload --- third_party/libgit2/lfs.patch | 86 ++++++++++++++++++++++++++++++++--- 1 file changed, 79 insertions(+), 7 deletions(-) diff --git a/third_party/libgit2/lfs.patch b/third_party/libgit2/lfs.patch index 8d4d43217e..d7244c377e 100644 --- a/third_party/libgit2/lfs.patch +++ b/third_party/libgit2/lfs.patch @@ -24,6 +24,25 @@ index 31da49a88..d61c9735e 100644 if(BUILD_EXAMPLES) add_subdirectory(examples) endif() +diff --git a/include/git2/oid.h b/include/git2/oid.h +index 0af9737a0..6d9a8b08a 100644 +--- a/include/git2/oid.h ++++ b/include/git2/oid.h +@@ -22,14 +22,8 @@ GIT_BEGIN_DECL + + /** The type of object id. */ + typedef enum { +- +-#ifdef GIT_EXPERIMENTAL_SHA256 + GIT_OID_SHA1 = 1, /**< SHA1 */ + GIT_OID_SHA256 = 2 /**< SHA256 */ +-#else +- GIT_OID_SHA1 = 1 /**< SHA1 */ +-#endif +- + } git_oid_t; + + /* diff --git a/include/git2/repository.h b/include/git2/repository.h index b203576af..26309dd3f 100644 --- a/include/git2/repository.h @@ -332,10 +351,10 @@ index 58cb4b424..00ddee9f3 100644 git_writestream **out, diff --git a/src/libgit2/lfs_filter.c b/src/libgit2/lfs_filter.c new file mode 100644 -index 000000000..6dc8f9650 +index 000000000..280bcc7ab --- /dev/null +++ b/src/libgit2/lfs_filter.c -@@ -0,0 +1,679 @@ +@@ -0,0 +1,732 @@ +/* +/ Copyright 2025 Intel Corporation +/ @@ -357,6 +376,7 @@ index 000000000..6dc8f9650 + +#include +#include "git2/sys/filter.h" ++#include "hash.h" +#include "oid.h" +#include "filter.h" +#include "str.h" @@ -466,6 +486,59 @@ index 000000000..6dc8f9650 + } +} + ++int git_oid_sha256_from_git_str_blob(git_oid *out, const struct git_str *input) ++{ ++ int error = -1; ++ git_hash_ctx *ctx = NULL; ++ ++ if (!out || !input || !input->ptr) ++ return -1; ++ ++ /* 1) Build "blob \\0" header (size = payload length in bytes) */ ++ char header[64]; /* plenty for "blob " + decimal size + \\0 */ ++ int hdrlen = snprintf(header, sizeof(header), "blob %zu", input->size); ++ if (hdrlen < 0 || (size_t)hdrlen + 1 >= sizeof(header)) ++ return -1; /* impossible header size for normal blob lengths */ ++ ++ git_hash_algorithm_t algorithm; ++ algorithm = git_oid_algorithm(GIT_OID_SHA256); ++ /* 2) Init SHA-256 hashing context (internal API) */ ++ if (git_hash_ctx_init(&ctx, algorithm) < 0) ++ goto done; ++ ++ /* 3) Feed header + NUL, then data */ ++ if (git_hash_update(ctx, header, (size_t)hdrlen) < 0) ++ goto done; ++ if (git_hash_update(ctx, "\0", 1) < 0) ++ goto done; ++ if (input->size > 0 && ++ git_hash_update(ctx, input->ptr, input->size) < 0) ++ goto done; ++ ++ /* 4) Finalize into git_oid (32-byte raw digest) */ ++ if (git_hash_final(out->id, ctx) < 0) ++ goto done; ++ ++ error = 0; ++ ++done: ++ if (ctx) ++ git_hash_ctx_cleanup(ctx); ++ return error; ++} ++ ++static int lfs_remove_id( ++ git_str *to, ++ const git_str *from) ++{ ++ git_oid lfs_oid; ++ git_oid_sha256_from_git_str_blob(&lfs_oid, from); ++ ++ fprintf("Size: %d", from->size); ++ fprintf("Oid sha256: %s", lfs_oid.id); ++ return 0; ++} ++ +static int lfs_insert_id( + git_str *to, const git_str *from, const git_filter_source *src, void** payload) +{ @@ -482,13 +555,11 @@ index 000000000..6dc8f9650 + const char *obj_regexp = "\noid sha256:(.*)\n"; + const char *size_regexp = "\nsize (.*)\n"; + -+ print_src_oid(src); + if (get_lfs_info_match(&lfs_oid, obj_regexp) < 0) { + fprintf(stderr,"\n[ERROR] failure, cannot find lfs oid in: %s\n", + lfs_oid.ptr); + return -1; + } -+ printf("\nfrom->lfs_oid %s\n", lfs_oid.ptr); + + if (get_lfs_info_match(&lfs_size, size_regexp) < 0) { + fprintf(stderr, @@ -497,6 +568,9 @@ index 000000000..6dc8f9650 + return -1; + } + ++ printf("\ndownload from->lfs_oid %s\n", lfs_oid.ptr); ++ printf("\ndownload from->lfs_size %s\n", lfs_size.ptr); ++ + git_repository *repo = git_filter_source_repo(src); + const char *path = git_filter_source_path(src); + @@ -533,10 +607,8 @@ index 000000000..6dc8f9650 + if (git_filter_source_mode(src) == GIT_FILTER_SMUDGE) + return lfs_insert_id(to, from, src, payload); + else -+ return GIT_PASSTHROUGH; -+ /* PATH for upload lfs files not needed ++ /* for upload of the lfs pointer files */ + return lfs_remove_id(to, from); -+ */ + return 0; +} + From 30072a9e9024ace837c4bcf879109f0a9c8f7a57 Mon Sep 17 00:00:00 2001 From: rasapala Date: Tue, 10 Mar 2026 11:46:00 +0100 Subject: [PATCH 20/33] Fix --- third_party/libgit2/lfs.patch | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/third_party/libgit2/lfs.patch b/third_party/libgit2/lfs.patch index d7244c377e..596c981ce6 100644 --- a/third_party/libgit2/lfs.patch +++ b/third_party/libgit2/lfs.patch @@ -351,7 +351,7 @@ index 58cb4b424..00ddee9f3 100644 git_writestream **out, diff --git a/src/libgit2/lfs_filter.c b/src/libgit2/lfs_filter.c new file mode 100644 -index 000000000..280bcc7ab +index 000000000..795f08b29 --- /dev/null +++ b/src/libgit2/lfs_filter.c @@ -0,0 +1,732 @@ @@ -534,8 +534,8 @@ index 000000000..280bcc7ab + git_oid lfs_oid; + git_oid_sha256_from_git_str_blob(&lfs_oid, from); + -+ fprintf("Size: %d", from->size); -+ fprintf("Oid sha256: %s", lfs_oid.id); ++ printf("Size: %d", from->size); ++ printf("Oid sha256: %s", lfs_oid.id); + return 0; +} + From 39661c8fb8b9c6051509e2a21038e226fe42bf33 Mon Sep 17 00:00:00 2001 From: rasapala Date: Tue, 10 Mar 2026 11:59:34 +0100 Subject: [PATCH 21/33] Fix 2 --- third_party/libgit2/lfs.patch | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/third_party/libgit2/lfs.patch b/third_party/libgit2/lfs.patch index 596c981ce6..9825cf3703 100644 --- a/third_party/libgit2/lfs.patch +++ b/third_party/libgit2/lfs.patch @@ -351,10 +351,10 @@ index 58cb4b424..00ddee9f3 100644 git_writestream **out, diff --git a/src/libgit2/lfs_filter.c b/src/libgit2/lfs_filter.c new file mode 100644 -index 000000000..795f08b29 +index 000000000..141d8e682 --- /dev/null +++ b/src/libgit2/lfs_filter.c -@@ -0,0 +1,732 @@ +@@ -0,0 +1,736 @@ +/* +/ Copyright 2025 Intel Corporation +/ @@ -532,10 +532,14 @@ index 000000000..795f08b29 + const git_str *from) +{ + git_oid lfs_oid; -+ git_oid_sha256_from_git_str_blob(&lfs_oid, from); ++ if (git_oid_sha256_from_git_str_blob(&lfs_oid, from) < 0) { ++ fprintf(stderr, ++ "\n[ERROR] failure, cannot calculate sha256: %s\n"); ++ return -1; ++ } + -+ printf("Size: %d", from->size); -+ printf("Oid sha256: %s", lfs_oid.id); ++ printf("\nSize: %d\n", from->size); ++ printf("\nOid sha256: %s\n", lfs_oid.id); + return 0; +} + From c2e8e8fdb80bf728752fc544bafbf633c816500e Mon Sep 17 00:00:00 2001 From: rasapala Date: Tue, 10 Mar 2026 12:15:12 +0100 Subject: [PATCH 22/33] Fix 3 --- third_party/libgit2/lfs.patch | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/third_party/libgit2/lfs.patch b/third_party/libgit2/lfs.patch index 9825cf3703..a71e15d9f8 100644 --- a/third_party/libgit2/lfs.patch +++ b/third_party/libgit2/lfs.patch @@ -351,10 +351,10 @@ index 58cb4b424..00ddee9f3 100644 git_writestream **out, diff --git a/src/libgit2/lfs_filter.c b/src/libgit2/lfs_filter.c new file mode 100644 -index 000000000..141d8e682 +index 000000000..6111fb556 --- /dev/null +++ b/src/libgit2/lfs_filter.c -@@ -0,0 +1,736 @@ +@@ -0,0 +1,734 @@ +/* +/ Copyright 2025 Intel Corporation +/ @@ -500,10 +500,8 @@ index 000000000..141d8e682 + if (hdrlen < 0 || (size_t)hdrlen + 1 >= sizeof(header)) + return -1; /* impossible header size for normal blob lengths */ + -+ git_hash_algorithm_t algorithm; -+ algorithm = git_oid_algorithm(GIT_OID_SHA256); + /* 2) Init SHA-256 hashing context (internal API) */ -+ if (git_hash_ctx_init(&ctx, algorithm) < 0) ++ if (git_hash_ctx_init(&ctx, GIT_HASH_ALGORITHM_SHA256) < 0) + goto done; + + /* 3) Feed header + NUL, then data */ @@ -534,7 +532,7 @@ index 000000000..141d8e682 + git_oid lfs_oid; + if (git_oid_sha256_from_git_str_blob(&lfs_oid, from) < 0) { + fprintf(stderr, -+ "\n[ERROR] failure, cannot calculate sha256: %s\n"); ++ "\n[ERROR] failure, cannot calculate sha256\n"); + return -1; + } + From ca6e1c128b33767564bb36e5f96ed2c683155a57 Mon Sep 17 00:00:00 2001 From: rasapala Date: Tue, 10 Mar 2026 16:54:58 +0100 Subject: [PATCH 23/33] Working sha --- third_party/libgit2/lfs.patch | 168 ++++++++++++++++++++++++++++------ 1 file changed, 141 insertions(+), 27 deletions(-) diff --git a/third_party/libgit2/lfs.patch b/third_party/libgit2/lfs.patch index a71e15d9f8..88d1b97e75 100644 --- a/third_party/libgit2/lfs.patch +++ b/third_party/libgit2/lfs.patch @@ -100,7 +100,7 @@ index d121c588a..b54a01a4b 100644 # diff --git a/src/cli/cmd_clone.c b/src/cli/cmd_clone.c -index c18cb28d4..286fa7153 100644 +index c18cb28d4..6d23dcbb1 100644 --- a/src/cli/cmd_clone.c +++ b/src/cli/cmd_clone.c @@ -146,6 +146,7 @@ int cmd_clone(int argc, char **argv) @@ -111,6 +111,76 @@ index c18cb28d4..286fa7153 100644 if (!checkout) clone_opts.checkout_opts.checkout_strategy = GIT_CHECKOUT_NONE; +@@ -182,6 +183,69 @@ int cmd_clone(int argc, char **argv) + + cli_progress_finish(&progress); + ++ ++ git_repository *repo2 = NULL; ++ int error = git_repository_open_ext(&repo2, local_path, 0, NULL); ++ // HEAD state info ++ bool is_detached = git_repository_head_detached(repo2) == 1; ++ bool is_unborn = git_repository_head_unborn(repo2) == 1; ++ ++ // Collect status (staged/unstaged/untracked) ++ git_status_options opts = GIT_STATUS_OPTIONS_INIT; ++ ++ opts.show = GIT_STATUS_SHOW_INDEX_AND_WORKDIR; ++ opts.flags = GIT_STATUS_OPT_INCLUDE_UNTRACKED // include untracked files ++ // // | ++ // GIT_STATUS_OPT_RENAMES_HEAD_TO_INDEX ++ // // detect renames ++ // HEAD->index - not ++ // required currently and ++ // impacts performance ++ | GIT_STATUS_OPT_SORT_CASE_SENSITIVELY; ++ ++ git_status_list *status_list = NULL; ++ ret = git_status_list_new(&status_list, repo2, &opts); ++ ++ size_t staged = 0, unstaged = 0, untracked = 0, conflicted = 0; ++ const size_t n = git_status_list_entrycount(status_list); ++ ++ for (size_t i = 0; i < n; ++i) { ++ const git_status_entry *e = git_status_byindex(status_list, i); ++ if (!e) ++ continue; ++ unsigned s = e->status; ++ ++ // Staged (index) changes ++ if (s & (GIT_STATUS_INDEX_NEW | GIT_STATUS_INDEX_MODIFIED | ++ GIT_STATUS_INDEX_DELETED | GIT_STATUS_INDEX_RENAMED | ++ GIT_STATUS_INDEX_TYPECHANGE)) ++ ++staged; ++ ++ // Unstaged (workdir) changes ++ if (s & (GIT_STATUS_WT_MODIFIED | GIT_STATUS_WT_DELETED | ++ GIT_STATUS_WT_RENAMED | GIT_STATUS_WT_TYPECHANGE)) ++ ++unstaged; ++ ++ // Untracked ++ if (s & GIT_STATUS_WT_NEW) ++ ++untracked; ++ ++ // Conflicted ++ if (s & GIT_STATUS_CONFLICTED) ++ ++conflicted; ++ } ++ ++ // Print summary (mirrors your original stream output) ++ printf("HEAD state : %s\n", ++ is_unborn ? "unborn (no commits)" : ++ (is_detached ? "detached" : "attached")); ++ printf("Staged changes : %zu\n", staged); ++ printf("Unstaged changes: %zu\n", unstaged); ++ printf("Untracked files : %zu", untracked); ++ if (conflicted) { ++ printf(" (%zu paths flagged)", conflicted); ++ } ++ printf("\n"); + done: + cli_progress_dispose(&progress); + git__free(computed_path); diff --git a/src/cli/progress.h b/src/cli/progress.h index f08d68f19..0344304ec 100644 --- a/src/cli/progress.h @@ -351,11 +421,11 @@ index 58cb4b424..00ddee9f3 100644 git_writestream **out, diff --git a/src/libgit2/lfs_filter.c b/src/libgit2/lfs_filter.c new file mode 100644 -index 000000000..6111fb556 +index 000000000..876628fd2 --- /dev/null +++ b/src/libgit2/lfs_filter.c -@@ -0,0 +1,734 @@ -+/* +@@ -0,0 +1,778 @@ ++/* +/ Copyright 2025 Intel Corporation +/ +/ Licensed under the Apache License, Version 2.0 (the "License"); @@ -486,42 +556,57 @@ index 000000000..6111fb556 + } +} + -+int git_oid_sha256_from_git_str_blob(git_oid *out, const struct git_str *input) ++int git_oid_sha256_from_git_str_blob( ++ git_oid *out, ++ const struct git_str *input, ++ char *pointer_line, ++ size_t pointer_line_cap) +{ + int error = -1; -+ git_hash_ctx *ctx = NULL; ++ git_hash_ctx ctx; + + if (!out || !input || !input->ptr) + return -1; + -+ /* 1) Build "blob \\0" header (size = payload length in bytes) */ -+ char header[64]; /* plenty for "blob " + decimal size + \\0 */ -+ int hdrlen = snprintf(header, sizeof(header), "blob %zu", input->size); -+ if (hdrlen < 0 || (size_t)hdrlen + 1 >= sizeof(header)) -+ return -1; /* impossible header size for normal blob lengths */ -+ -+ /* 2) Init SHA-256 hashing context (internal API) */ ++ /* 1) Init SHA-256 hashing context (internal API) */ + if (git_hash_ctx_init(&ctx, GIT_HASH_ALGORITHM_SHA256) < 0) + goto done; + -+ /* 3) Feed header + NUL, then data */ -+ if (git_hash_update(ctx, header, (size_t)hdrlen) < 0) -+ goto done; -+ if (git_hash_update(ctx, "\0", 1) < 0) -+ goto done; -+ if (input->size > 0 && -+ git_hash_update(ctx, input->ptr, input->size) < 0) -+ goto done; ++ /* 2) Stream the payload in chunks — hash *only* the file bytes. */ ++ const size_t CHUNK = 4 * 1024 * 1024; /* 4 MiB */ ++ const unsigned char *p = (const unsigned char *)input->ptr; ++ size_t remaining = input->size; ++ ++ while (remaining > 0) { ++ size_t n = remaining > CHUNK ? CHUNK : remaining; ++ if (git_hash_update(&ctx, p, n) < 0) ++ goto done; ++ p += n; ++ remaining -= n; ++ } + -+ /* 4) Finalize into git_oid (32-byte raw digest) */ -+ if (git_hash_final(out->id, ctx) < 0) ++ /* 3) Finalize into git_oid (32-byte raw digest for SHA-256). */ ++ if (git_hash_final(out->id, &ctx) < 0) + goto done; + ++ /* 4) Optionally format "oid sha256:" for the LFS pointer file. */ ++ if (pointer_line && ++ pointer_line_cap >= (size_t)(strlen("oid sha256:") + 64 + 1)) { ++ char hex[64 + 1]; ++ /* Formats full hex; no NUL added. */ ++ if (git_oid_fmt(hex, out) < 0) { ++ fprintf(stderr, ++ "\n[ERROR] failure, git_oid_fmt failed\n"); ++ return -1; ++ } ++ ++ hex[64] = '\0'; ++ snprintf(pointer_line, pointer_line_cap, "oid sha256:%s", hex); ++ } ++ + error = 0; + +done: -+ if (ctx) -+ git_hash_ctx_cleanup(ctx); + return error; +} + @@ -529,15 +614,44 @@ index 000000000..6111fb556 + git_str *to, + const git_str *from) +{ ++ int error = 0; ++ ++ if(!from) return -1; ++ ++ /* Use lib git oid to get lfs sha256 */ + git_oid lfs_oid; -+ if (git_oid_sha256_from_git_str_blob(&lfs_oid, from) < 0) { ++ lfs_oid.type = GIT_OID_SHA256; ++ char line[80]; /* 75+ is enough */ ++ if (git_oid_sha256_from_git_str_blob(&lfs_oid, from, line, sizeof(line)) < 0) ++ { + fprintf(stderr, + "\n[ERROR] failure, cannot calculate sha256\n"); + return -1; + } + + printf("\nSize: %d\n", from->size); -+ printf("\nOid sha256: %s\n", lfs_oid.id); ++ printf("\nOid sha256: %s\n", line); ++ ++ git_str_init(to, 0); ++ ++ /* 1) version line (LFS spec requires this literal string) */ ++ if ((error = git_str_puts(to, "version https://git-lfs.github.com/spec/v1\n")) < 0) ++ return error; ++ ++ ++ /* 2) the oid line passed by caller (must end with '\n') */ ++ if ((error = git_str_puts(to, line)) < 0) ++ return error; ++ ++ if (line[strlen(line) - 1] != '\n') { ++ if ((error = git_str_putc(to, '\n')) < 0) ++ return error; ++ } ++ ++ /* 3) size line from the original file size */ ++ if ((error = git_str_printf(to, "size %zu\n", from->size)) < 0) ++ return error; ++ + return 0; +} + From 110c4e2fdb7dca32437b5c45e1b0cb3ffc13ad76 Mon Sep 17 00:00:00 2001 From: rasapala Date: Wed, 11 Mar 2026 10:01:54 +0100 Subject: [PATCH 24/33] Experimentasl cmake off --- third_party/libgit2/lfs.patch | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/third_party/libgit2/lfs.patch b/third_party/libgit2/lfs.patch index 88d1b97e75..300a6ef986 100644 --- a/third_party/libgit2/lfs.patch +++ b/third_party/libgit2/lfs.patch @@ -24,6 +24,17 @@ index 31da49a88..d61c9735e 100644 if(BUILD_EXAMPLES) add_subdirectory(examples) endif() +diff --git a/cmake/ExperimentalFeatures.cmake b/cmake/ExperimentalFeatures.cmake +index 7eff40bdb..5562acc77 100644 +--- a/cmake/ExperimentalFeatures.cmake ++++ b/cmake/ExperimentalFeatures.cmake +@@ -18,6 +18,3 @@ else() + add_feature_info("SHA256 API" OFF "experimental SHA256 APIs") + endif() + +-if(EXPERIMENTAL) +- set(LIBGIT2_FILENAME "${LIBGIT2_FILENAME}-experimental") +-endif() diff --git a/include/git2/oid.h b/include/git2/oid.h index 0af9737a0..6d9a8b08a 100644 --- a/include/git2/oid.h From 18c37bf25261d47cc5cdc5dd8233bb3d9cd69f77 Mon Sep 17 00:00:00 2001 From: Rafal Sapala Date: Wed, 11 Mar 2026 10:36:11 +0100 Subject: [PATCH 25/33] Experimental flag --- third_party/libgit2/libgit2_engine.bzl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/third_party/libgit2/libgit2_engine.bzl b/third_party/libgit2/libgit2_engine.bzl index 68a5037aff..5df5667a5c 100644 --- a/third_party/libgit2/libgit2_engine.bzl +++ b/third_party/libgit2/libgit2_engine.bzl @@ -51,6 +51,7 @@ def _impl(repository_ctx): out_static = "out_interface_libs = [\"{lib_name}.lib\"],".format(lib_name=lib_name) out_libs = "out_shared_libs = [\"{lib_name}.dll\"],".format(lib_name=lib_name) cache_entries = """ + "EXPERIMENTAL_SHA256": "ON", "CMAKE_POSITION_INDEPENDENT_CODE": "ON", "CMAKE_CXX_FLAGS": " /guard:cf /GS -s -D_GLIBCXX_USE_CXX11_ABI=1", "CMAKE_LIBRARY_OUTPUT_DIRECTORY": "Debug", @@ -66,6 +67,7 @@ def _impl(repository_ctx): out_static = "" out_libs = "out_shared_libs = [\"{lib_name}.so\"],".format(lib_name=lib_name) cache_entries = """ + "EXPERIMENTAL_SHA256": "ON", "CMAKE_POSITION_INDEPENDENT_CODE": "ON", "CMAKE_CXX_FLAGS": " /guard:cf -s -D_GLIBCXX_USE_CXX11_ABI=1 -Wno-error=deprecated-declarations -Wuninitialized", "CMAKE_ARCHIVE_OUTPUT_DIRECTORY": "lib", From 2bae83dbcf18b33f41cb9342715a8e8885ecc9e9 Mon Sep 17 00:00:00 2001 From: rasapala Date: Wed, 11 Mar 2026 10:36:45 +0100 Subject: [PATCH 26/33] Cleanup --- third_party/libgit2/lfs.patch | 94 +++++++++++++++++++++++------------ 1 file changed, 63 insertions(+), 31 deletions(-) diff --git a/third_party/libgit2/lfs.patch b/third_party/libgit2/lfs.patch index 300a6ef986..b6daae42ae 100644 --- a/third_party/libgit2/lfs.patch +++ b/third_party/libgit2/lfs.patch @@ -432,10 +432,10 @@ index 58cb4b424..00ddee9f3 100644 git_writestream **out, diff --git a/src/libgit2/lfs_filter.c b/src/libgit2/lfs_filter.c new file mode 100644 -index 000000000..876628fd2 +index 000000000..fc3120cca --- /dev/null +++ b/src/libgit2/lfs_filter.c -@@ -0,0 +1,778 @@ +@@ -0,0 +1,810 @@ +/* +/ Copyright 2025 Intel Corporation +/ @@ -473,6 +473,7 @@ index 000000000..876628fd2 + const char *lfs_oid; + const char *lfs_size; + const char *url; ++ bool is_download; +} lfs_attrs; + +static size_t get_digit(const char *buffer) @@ -576,12 +577,20 @@ index 000000000..876628fd2 + int error = -1; + git_hash_ctx ctx; + -+ if (!out || !input || !input->ptr) ++ if (!out || !input || !input->ptr) { + return -1; ++ } ++ ++ if (!pointer_line || ++ pointer_line_cap < (size_t)(strlen("oid sha256:") + 64 + 1)) { ++ return -1; ++ } + + /* 1) Init SHA-256 hashing context (internal API) */ -+ if (git_hash_ctx_init(&ctx, GIT_HASH_ALGORITHM_SHA256) < 0) -+ goto done; ++ if (git_hash_ctx_init(&ctx, GIT_HASH_ALGORITHM_SHA256) < 0) { ++ fprintf(stderr, "\n[ERROR] git_hash_ctx_init failed\n"); ++ return -1; ++ } + + /* 2) Stream the payload in chunks — hash *only* the file bytes. */ + const size_t CHUNK = 4 * 1024 * 1024; /* 4 MiB */ @@ -590,15 +599,19 @@ index 000000000..876628fd2 + + while (remaining > 0) { + size_t n = remaining > CHUNK ? CHUNK : remaining; -+ if (git_hash_update(&ctx, p, n) < 0) -+ goto done; ++ if (git_hash_update(&ctx, p, n) < 0) { ++ fprintf(stderr, "\n[ERROR] git_hash_update failed\n"); ++ return -1; ++ } + p += n; + remaining -= n; + } + + /* 3) Finalize into git_oid (32-byte raw digest for SHA-256). */ -+ if (git_hash_final(out->id, &ctx) < 0) -+ goto done; ++ if (git_hash_final(out->id, &ctx) < 0) { ++ fprintf(stderr, "\n[ERROR] git_hash_final failed\n"); ++ return -1; ++ } + + /* 4) Optionally format "oid sha256:" for the LFS pointer file. */ + if (pointer_line && @@ -615,53 +628,67 @@ index 000000000..876628fd2 + snprintf(pointer_line, pointer_line_cap, "oid sha256:%s", hex); + } + -+ error = 0; -+ -+done: -+ return error; ++ return 0; +} + +static int lfs_remove_id( + git_str *to, -+ const git_str *from) ++ const git_str *from, ++ void **payload) +{ + int error = 0; ++ /* Init the lfs attrs to indicate git lfs clean, currently only diff support no upload of lfs file supported */ ++ struct lfs_attrs la = { NULL, NULL, NULL, NULL, NULL, NULL, false }; ++ *payload = git__malloc(sizeof(la)); ++ GIT_ERROR_CHECK_ALLOC(*payload); ++ memcpy(*payload, &la, sizeof(la)); + + if(!from) return -1; + ++ /* lfs spec - return empty pointer when the file is empty */ ++ if (from->size == 0) { ++ git_str_init(to, 0); ++ return 0; ++ } ++ + /* Use lib git oid to get lfs sha256 */ + git_oid lfs_oid; + lfs_oid.type = GIT_OID_SHA256; + char line[80]; /* 75+ is enough */ -+ if (git_oid_sha256_from_git_str_blob(&lfs_oid, from, line, sizeof(line)) < 0) -+ { ++ if (git_oid_sha256_from_git_str_blob(&lfs_oid, from, line, sizeof(line)) < 0) { + fprintf(stderr, + "\n[ERROR] failure, cannot calculate sha256\n"); + return -1; + } + -+ printf("\nSize: %d\n", from->size); -+ printf("\nOid sha256: %s\n", line); -+ + git_str_init(to, 0); + + /* 1) version line (LFS spec requires this literal string) */ -+ if ((error = git_str_puts(to, "version https://git-lfs.github.com/spec/v1\n")) < 0) -+ return error; ++ if ((error = git_str_puts( ++ to, "version https://git-lfs.github.com/spec/v1\n")) < 0) { ++ fprintf(stderr, "\n[ERROR] git_str_puts failed\n"); ++ return error; ++ } ++ + -+ + /* 2) the oid line passed by caller (must end with '\n') */ -+ if ((error = git_str_puts(to, line)) < 0) ++ if ((error = git_str_puts(to, line)) < 0) { ++ fprintf(stderr, "\n[ERROR] git_str_puts failed\n"); + return error; ++ } + + if (line[strlen(line) - 1] != '\n') { -+ if ((error = git_str_putc(to, '\n')) < 0) ++ if ((error = git_str_putc(to, '\n')) < 0) { ++ fprintf(stderr, "\n[ERROR] git_str_putc failed\n"); + return error; ++ } + } + + /* 3) size line from the original file size */ -+ if ((error = git_str_printf(to, "size %zu\n", from->size)) < 0) ++ if ((error = git_str_printf(to, "size %zu\n", from->size)) < 0) { ++ fprintf(stderr, "\n[ERROR] git_str_printf failed\n"); + return error; ++ } + + return 0; +} @@ -695,9 +722,6 @@ index 000000000..876628fd2 + return -1; + } + -+ printf("\ndownload from->lfs_oid %s\n", lfs_oid.ptr); -+ printf("\ndownload from->lfs_size %s\n", lfs_size.ptr); -+ + git_repository *repo = git_filter_source_repo(src); + const char *path = git_filter_source_path(src); + @@ -712,7 +736,7 @@ index 000000000..876628fd2 + size_t workdir_size = strlen(git_repository_workdir(repo)); + + const char *workdir = git_repository_workdir(repo); -+ struct lfs_attrs la = { path, full_path.ptr, workdir, lfs_oid.ptr, lfs_size.ptr, repo->url }; ++ struct lfs_attrs la = { path, full_path.ptr, workdir, lfs_oid.ptr, lfs_size.ptr, repo->url, true }; + + *payload = git__malloc(sizeof(la)); + GIT_ERROR_CHECK_ALLOC(*payload); @@ -731,11 +755,12 @@ index 000000000..876628fd2 +{ + GIT_UNUSED(self); GIT_UNUSED(payload); + ++ /* for download of the lfs pointer files */ + if (git_filter_source_mode(src) == GIT_FILTER_SMUDGE) + return lfs_insert_id(to, from, src, payload); + else -+ /* for upload of the lfs pointer files */ -+ return lfs_remove_id(to, from); ++ /* for upload or diff of the lfs pointer files */ ++ return lfs_remove_id(to, from, payload); + return 0; +} + @@ -993,6 +1018,13 @@ index 000000000..876628fd2 + return; + } + struct lfs_attrs *la = (struct lfs_attrs *)payload; ++ ++ /* Currently only download is supoprted, no lfs file upload */ ++ if (!la->is_download) { ++ git__free(payload); ++ return; ++ } ++ + char *tmp_out_file = append_cstr_to_buffer(la->full_path, "lfs_part"); + + CURL *info_curl,*dl_curl; From ad313b772bfd4cfdc852277e8c3b630f470ec52e Mon Sep 17 00:00:00 2001 From: Rafal Sapala Date: Wed, 11 Mar 2026 11:34:08 +0100 Subject: [PATCH 27/33] Cleanup --- src/pull_module/libgit2.cpp | 332 +++++++++++++++++++++----------- src/pull_module/libgit2.hpp | 1 - src/status.cpp | 1 - src/test/pull_hf_model_test.cpp | 21 +- 4 files changed, 233 insertions(+), 122 deletions(-) diff --git a/src/pull_module/libgit2.cpp b/src/pull_module/libgit2.cpp index 570ecd08fe..14f7bcdb89 100644 --- a/src/pull_module/libgit2.cpp +++ b/src/pull_module/libgit2.cpp @@ -15,9 +15,11 @@ //***************************************************************************** #include "libgit2.hpp" +#include #include -#include #include +#include +#include #include #include @@ -69,16 +71,16 @@ int cred_acquire_cb(git_credential** out, password = _strdup(username); #endif } else { - fprintf(stderr, "HF_TOKEN env variable is not set.\n"); + fprintf(stderr, "[ERROR] HF_TOKEN env variable is not set.\n"); return -1; } error = git_credential_userpass_plaintext_new(out, username, password); if (error < 0) { - fprintf(stderr, "Creating credentials failed.\n"); + fprintf(stderr, "[ERROR] Creating credentials failed.\n"); error = -1; } } else { - fprintf(stderr, "Only USERPASS_PLAINTEXT supported in OVMS.\n"); + fprintf(stderr, "[ERROR] Only USERPASS_PLAINTEXT supported in OVMS.\n"); return 1; } @@ -180,56 +182,94 @@ Status HfDownloader::RemoveReadonlyFileAttributeFromDir(const std::string& direc return StatusCode::OK; } -Status HfDownloader::CheckRepositoryStatus(bool checkUntracked) { - git_repository *repo = NULL; - int error = git_repository_open_ext(&repo, this->downloadPath.c_str(), 0, NULL); - if (error < 0) { - const git_error *err = git_error_last(); - if (err) - SPDLOG_ERROR("Repository open failed: {} {}", err->klass, err->message); - else - SPDLOG_ERROR("Repository open failed: {}", error); - if (repo) git_repository_free(repo); +class GitRepositoryGuard { +public: + git_repository* repo = nullptr; + + GitRepositoryGuard(const std::string& path) { + int error = git_repository_open_ext(&repo, path.c_str(), 0, nullptr); + if (error < 0) { + const git_error* err = git_error_last(); + if (err) + SPDLOG_ERROR("Repository open failed: {} {}", err->klass, err->message); + else + SPDLOG_ERROR("Repository open failed: {}", error); + if (repo) + git_repository_free(repo); + } + } + + ~GitRepositoryGuard() { + if (repo) { + git_repository_free(repo); + } + } + // Allow implicit access to the raw pointer + git_repository* get() const { return repo; } + operator git_repository*() const { return repo; } + + // Non-copyable + GitRepositoryGuard(const GitRepositoryGuard&) = delete; + GitRepositoryGuard& operator=(const GitRepositoryGuard&) = delete; + + // Movable + GitRepositoryGuard(GitRepositoryGuard&& other) noexcept { + repo = other.repo; + other.repo = nullptr; + } + GitRepositoryGuard& operator=(GitRepositoryGuard&& other) noexcept { + if (this != &other) { + if (repo) + git_repository_free(repo); + repo = other.repo; + other.repo = nullptr; + } + return *this; + } +}; + +Status HfDownloader::CheckRepositoryStatus(bool checkUntracked) { + GitRepositoryGuard repoGuard(this->downloadPath); + if (!repoGuard.get()) { return StatusCode::HF_GIT_STATUS_FAILED; } // HEAD state info - bool is_detached = git_repository_head_detached(repo) == 1; - bool is_unborn = git_repository_head_unborn(repo) == 1; - + bool is_detached = git_repository_head_detached(repoGuard.get()) == 1; + bool is_unborn = git_repository_head_unborn(repoGuard.get()) == 1; + // Collect status (staged/unstaged/untracked) git_status_options opts = GIT_STATUS_OPTIONS_INIT; - - opts.show = GIT_STATUS_SHOW_INDEX_AND_WORKDIR; - opts.flags = GIT_STATUS_OPT_INCLUDE_UNTRACKED // include untracked files // | GIT_STATUS_OPT_RENAMES_HEAD_TO_INDEX // detect renames HEAD->index - not required currently and impacts performance - | GIT_STATUS_OPT_SORT_CASE_SENSITIVELY; - + opts.show = GIT_STATUS_SHOW_INDEX_AND_WORKDIR; + opts.flags = GIT_STATUS_OPT_INCLUDE_UNTRACKED // include untracked files // | GIT_STATUS_OPT_RENAMES_HEAD_TO_INDEX // detect renames HEAD->index - not required currently and impacts performance + | GIT_STATUS_OPT_SORT_CASE_SENSITIVELY; + git_status_list* status_list = nullptr; - error = git_status_list_new(&status_list, repo, &opts); + int error = git_status_list_new(&status_list, repoGuard.get(), &opts); if (error != 0) { return StatusCode::HF_GIT_STATUS_FAILED; } size_t staged = 0, unstaged = 0, untracked = 0, conflicted = 0; - const size_t n = git_status_list_entrycount(status_list); // iterate entries + const size_t n = git_status_list_entrycount(status_list); // iterate entries for (size_t i = 0; i < n; ++i) { const git_status_entry* e = git_status_byindex(status_list, i); unsigned s = e->status; // Staged (index) changes - if (s & (GIT_STATUS_INDEX_NEW | - GIT_STATUS_INDEX_MODIFIED| - GIT_STATUS_INDEX_DELETED | - GIT_STATUS_INDEX_RENAMED | - GIT_STATUS_INDEX_TYPECHANGE)) + if (s & (GIT_STATUS_INDEX_NEW | + GIT_STATUS_INDEX_MODIFIED | + GIT_STATUS_INDEX_DELETED | + GIT_STATUS_INDEX_RENAMED | + GIT_STATUS_INDEX_TYPECHANGE)) ++staged; // Unstaged (workdir) changes - if (s & (GIT_STATUS_WT_MODIFIED | - GIT_STATUS_WT_DELETED | - GIT_STATUS_WT_RENAMED | - GIT_STATUS_WT_TYPECHANGE)) + if (s & (GIT_STATUS_WT_MODIFIED | + GIT_STATUS_WT_DELETED | + GIT_STATUS_WT_RENAMED | + GIT_STATUS_WT_TYPECHANGE)) ++unstaged; // Untracked @@ -243,46 +283,52 @@ Status HfDownloader::CheckRepositoryStatus(bool checkUntracked) { std::stringstream ss; ss << "HEAD state : " - << (is_unborn ? "unborn (no commits)" : (is_detached ? "detached" : "attached")) - << "\n"; - ss << "Staged changes : " << staged << "\n"; - ss << "Unstaged changes: " << unstaged << "\n"; - ss << "Untracked files : " << untracked << "\n"; - if (conflicted) ss << " (" << conflicted << " paths flagged)"; + << (is_unborn ? "unborn (no commits)" : (is_detached ? "detached" : "attached")) + << "\n"; + ss << "Staged changes : " << staged << "\n"; + ss << "Unstaged changes: " << unstaged << "\n"; + ss << "Untracked files : " << untracked << "\n"; + if (conflicted) + ss << " (" << conflicted << " paths flagged)"; SPDLOG_DEBUG(ss.str()); git_status_list_free(status_list); // We do not care about untracked until after git clone if (is_unborn || is_detached || staged || unstaged || conflicted || (checkUntracked && untracked)) { - return StatusCode::HF_GIT_STATUS_UNCLEAN; + return StatusCode::HF_GIT_STATUS_UNCLEAN; } return StatusCode::OK; } -#define CHECK(call) do { \ - int _err = (call); \ - if (_err < 0) { \ - const git_error *e = git_error_last(); \ - fprintf(stderr, "Error %d: %s (%s:%d)\n", _err, e && e->message ? e->message : "no message", __FILE__, __LINE__); \ - return; \ - } \ -} while (0) +#define CHECK(call) \ + do { \ + int _err = (call); \ + if (_err < 0) { \ + const git_error* e = git_error_last(); \ + fprintf(stderr, "[ERROR] %d: %s (%s:%d)\n", _err, e && e->message ? e->message : "no message", __FILE__, __LINE__); \ + return; \ + } \ + } while (0) // Trim trailing '\r' (for CRLF files) and surrounding spaces static inline void rtrimCrLfWhitespace(std::string& s) { - if (!s.empty() && s.back() == '\r') s.pop_back(); // remove trailing '\r' - while (!s.empty() && std::isspace(static_cast(s.back()))) s.pop_back(); // trailing ws + if (!s.empty() && s.back() == '\r') + s.pop_back(); // remove trailing '\r' + while (!s.empty() && std::isspace(static_cast(s.back()))) + s.pop_back(); // trailing ws size_t i = 0; - while (i < s.size() && std::isspace(static_cast(s[i]))) ++i; // leading ws - if (i > 0) s.erase(0, i); + while (i < s.size() && std::isspace(static_cast(s[i]))) + ++i; // leading ws + if (i > 0) + s.erase(0, i); } // Case-insensitive substring search: returns true if 'needle' is found in 'hay' static bool containsCaseInsensitive(const std::string& hay, const std::string& needle) { auto toLower = [](std::string v) { std::transform(v.begin(), v.end(), v.begin(), - [](unsigned char c){ return static_cast(std::tolower(c)); }); + [](unsigned char c) { return static_cast(std::tolower(c)); }); return v; }; std::string hayLower = toLower(hay); @@ -295,7 +341,8 @@ static bool containsCaseInsensitive(const std::string& hay, const std::string& n static bool readFirstThreeLines(const fs::path& p, std::vector& outLines) { outLines.clear(); std::ifstream in(p, std::ios::in | std::ios::binary); - if (!in) return false; + if (!in) + return false; constexpr std::streamsize kMaxPerLine = 8192; @@ -307,14 +354,18 @@ static bool readFirstThreeLines(const fs::path& p, std::vector& out char ch; bool gotNewline = false; while (count < kMaxPerLine && in.get(ch)) { - if (ch == '\n') { gotNewline = true; break; } + if (ch == '\n') { + gotNewline = true; + break; + } line.push_back(ch); ++count; } // If we hit kMaxPerLine without encountering '\n', drain until newline to resync if (count == kMaxPerLine && !gotNewline) { while (in.get(ch)) { - if (ch == '\n') break; + if (ch == '\n') + break; } } @@ -324,7 +375,8 @@ static bool readFirstThreeLines(const fs::path& p, std::vector& out } rtrimCrLfWhitespace(line); outLines.push_back(line); - if (!in) break; // Handle EOF gracefully + if (!in) + break; // Handle EOF gracefully } return true; } @@ -333,37 +385,42 @@ static bool readFirstThreeLines(const fs::path& p, std::vector& out // line1 -> "version", line2 -> "oid", line3 -> "size" (case-insensitive). static bool fileHasLfsKeywordsFirst3Positional(const fs::path& p) { std::error_code ec; - if (!fs::is_regular_file(p, ec)) return false; + if (!fs::is_regular_file(p, ec)) + return false; std::vector lines; - if (!readFirstThreeLines(p, lines)) return false; + if (!readFirstThreeLines(p, lines)) + return false; - if (lines.size() < 3) return false; + if (lines.size() < 3) + return false; return containsCaseInsensitive(lines[0], "version") && containsCaseInsensitive(lines[1], "oid") && containsCaseInsensitive(lines[2], "size"); } - // Helper: make path relative to base (best-effort, non-throwing). static fs::path makeRelativeToBase(const fs::path& path, const fs::path& base) { std::error_code ec; // Try fs::relative first (handles canonical comparisons, may fail if on different roots) fs::path rel = fs::relative(path, base, ec); - if (!ec && !rel.empty()) return rel; + if (!ec && !rel.empty()) + return rel; // Fallback: purely lexical relative (doesn't access filesystem) rel = path.lexically_relative(base); - if (!rel.empty()) return rel; + if (!rel.empty()) + return rel; // Last resort: return filename only (better than absolute when nothing else works) - if (path.has_filename()) return path.filename(); + if (path.has_filename()) + return path.filename(); return path; } // Find all files under 'directory' that satisfy the first-3-lines LFS keyword check. -std::vector findLfsLikeFiles(const std::string& directory, bool recursive = true) { +static std::vector findLfsLikeFiles(const std::string& directory, bool recursive = true) { std::vector matches; std::error_code ec; @@ -389,38 +446,102 @@ std::vector findLfsLikeFiles(const std::string& directory, bool recurs return matches; } -void resumeLfsDownloadForFile(git_repository *repo, const char *file_path_in_repo) { - git_object *obj = NULL; - git_tree *tree = NULL; - git_tree_entry *entry = NULL; - git_blob *blob = NULL; - git_buf out = GIT_BUF_INIT; +// pick the right entry pointer type for your libgit2 +#if defined(GIT_LIBGIT2_VER_MAJOR) +// libgit2 ≥ 1.0 generally has const-correct free() (accepts const*) +using git_tree_entry_ptr = const git_tree_entry*; +#else +using git_tree_entry_ptr = git_tree_entry*; +#endif + +// Single guard that owns all temporaries used in resumeLfsDownloadForFile +struct GitScope { + git_object* tree_obj = nullptr; // owns the tree as a generic git_object + git_tree_entry_ptr entry = nullptr; // owns the entry + git_blob* blob = nullptr; // owns the blob + git_buf out = GIT_BUF_INIT; // owns the buffer + + GitScope() = default; + ~GitScope() { cleanup(); } + + GitScope(const GitScope&) = delete; + GitScope& operator=(const GitScope&) = delete; + + GitScope(GitScope&& other) noexcept : + tree_obj(other.tree_obj), + entry(other.entry), + blob(other.blob), + out(other.out) { + other.tree_obj = nullptr; + other.entry = nullptr; + other.blob = nullptr; + other.out = GIT_BUF_INIT; + } + GitScope& operator=(GitScope&& other) noexcept { + if (this != &other) { + cleanup(); + tree_obj = other.tree_obj; + entry = other.entry; + blob = other.blob; + out = other.out; + other.tree_obj = nullptr; + other.entry = nullptr; + other.blob = nullptr; + other.out = GIT_BUF_INIT; + } + return *this; + } + + git_tree* tree() const { return reinterpret_cast(tree_obj); } + +private: + void cleanup() noexcept { + git_buf_dispose(&out); + if (blob) { + git_blob_free(blob); + blob = nullptr; + } + if (entry) { + git_tree_entry_free(entry); + entry = nullptr; + } + if (tree_obj) { + git_object_free(tree_obj); + tree_obj = nullptr; + } + } +}; + +void resumeLfsDownloadForFile(git_repository* repo, const char* filePathInRepo) { + GitScope g; + + // Resolve HEAD tree (origin/main^{tree}) + CHECK(git_revparse_single(&g.tree_obj, repo, "origin/main^{tree}")); + + // Find the tree entry by path + CHECK(git_tree_entry_bypath(&g.entry, g.tree(), filePathInRepo)); + + // Ensure it's a blob + if (git_tree_entry_type(g.entry) != GIT_OBJECT_BLOB) { + fprintf(stderr, "[ERROR] Path is not a blob: %s\n", filePathInRepo); + return; // Guard cleans up + } + + // Lookup the blob + CHECK(git_blob_lookup(&g.blob, repo, git_tree_entry_id(g.entry))); + // Configure filter behavior git_blob_filter_options opts = GIT_BLOB_FILTER_OPTIONS_INIT; // Choose direction: // GIT_BLOB_FILTER_TO_WORKTREE : apply smudge (as if writing to working tree) // GIT_BLOB_FILTER_TO_ODB : apply clean (as if writing to ODB) - // opts.flags = GIT_FILTER_TO_WORKTREE; + // opts.flags = GIT_BLOB_FILTER_TO_WORKTREE; - // Resolve HEAD tree - CHECK(git_revparse_single(&obj, repo, "origin/main^{tree}") != 0); - tree = (git_tree *)obj; + // Apply filters based on .gitattributes for this path (triggers LFS smudge/clean) + CHECK(git_blob_filter(&g.out, g.blob, filePathInRepo, &opts)); - // Find the tree entry and get the blob - CHECK(git_tree_entry_bypath(&entry, tree, file_path_in_repo) != 0); - CHECK(git_tree_entry_type(entry) != GIT_OBJECT_BLOB); - - CHECK(git_blob_lookup(&blob, repo, git_tree_entry_id(entry)) != 0); - - // Apply filters based on .gitattributes for this path - CHECK(git_blob_filter(&out, blob, file_path_in_repo, &opts) != 0); - - git_buf_dispose(&out); - if (blob) git_blob_free(blob); - if (entry) git_tree_entry_free(entry); - if (tree) git_tree_free(tree); - if (obj) git_object_free(obj); - return; + // We don't need the buffer contents; the filter side-effects are enough. + // All resources (out, blob, entry, tree_obj) will be freed automatically here. } Status HfDownloader::downloadModel() { @@ -433,7 +554,7 @@ Status HfDownloader::downloadModel() { if (std::filesystem::is_directory(this->downloadPath) && !this->overwriteModels) { // Checking if the download was partially finished for any files in repository auto matches = findLfsLikeFiles(this->downloadPath, true); - + if (matches.empty()) { std::cout << "No files to resume download found.\n"; std::cout << "Path already exists on local filesystem. Skipping download to path: " << this->downloadPath << std::endl; @@ -445,40 +566,31 @@ Status HfDownloader::downloadModel() { } } - git_repository *repo = NULL; - int error = git_repository_open_ext(&repo, this->downloadPath.c_str(), 0, NULL); - if (error < 0) { - const git_error *err = git_error_last(); - if (err) - SPDLOG_ERROR("Repository open failed: {} {}", err->klass, err->message); - else - SPDLOG_ERROR("Repository open failed: {}", error); - if (repo) git_repository_free(repo); - + GitRepositoryGuard repoGuard(this->downloadPath); + if (!repoGuard.get()) { std::cout << "Path already exists on local filesystem. And is not a git repository: " << this->downloadPath << std::endl; - return StatusCode::HF_GIT_CLONE_FAILED; + return StatusCode::HF_GIT_STATUS_FAILED; } // Set repository url std::string passRepoUrl = GetRepositoryUrlWithPassword(); const char* url = passRepoUrl.c_str(); - error = git_repository_set_url(repo, url); + int error = git_repository_set_url(repoGuard.get(), url); if (error < 0) { - const git_error *err = git_error_last(); + const git_error* err = git_error_last(); if (err) SPDLOG_ERROR("Repository set url failed: {} {}", err->klass, err->message); else SPDLOG_ERROR("Repository set url failed: {}", error); - if (repo) git_repository_free(repo); std::cout << "Path already exists on local filesystem. And set git repository url failed: " << this->downloadPath << std::endl; return StatusCode::HF_GIT_CLONE_FAILED; } for (const auto& p : matches) { - std::cout << " Resuming " << p.string() << "...\n"; - std::string path = p.string(); - resumeLfsDownloadForFile(repo, path.c_str()); - } + std::cout << " Resuming " << p.string() << "...\n"; + std::string path = p.string(); + resumeLfsDownloadForFile(repoGuard.get(), path.c_str()); + } SPDLOG_DEBUG("Checking repository status."); auto status = CheckRepositoryStatus(false); @@ -495,6 +607,7 @@ Status HfDownloader::downloadModel() { } SPDLOG_DEBUG("Downloading to path: {}", this->downloadPath); + git_repository* cloned_repo = NULL; // clone_opts for progress reporting set in libgit2 lib by patch git_clone_options clone_opts = GIT_CLONE_OPTIONS_INIT; @@ -523,7 +636,6 @@ Status HfDownloader::downloadModel() { SPDLOG_ERROR("Libgit2 clone error: {} message: {}", err->klass, err->message); else SPDLOG_ERROR("Libgit2 clone error: {}", error); - return StatusCode::HF_GIT_CLONE_FAILED; } else if (cloned_repo) { git_repository_free(cloned_repo); diff --git a/src/pull_module/libgit2.hpp b/src/pull_module/libgit2.hpp index b8dacac0e9..fb5549b16d 100644 --- a/src/pull_module/libgit2.hpp +++ b/src/pull_module/libgit2.hpp @@ -63,6 +63,5 @@ class HfDownloader : public IModelDownloader { bool CheckIfProxySet(); Status RemoveReadonlyFileAttributeFromDir(const std::string& directoryPath); Status CheckRepositoryStatus(bool checkUntracked); - int CheckRepositoryForResume(); }; } // namespace ovms diff --git a/src/status.cpp b/src/status.cpp index b83b4f7a45..a7750498e9 100644 --- a/src/status.cpp +++ b/src/status.cpp @@ -350,7 +350,6 @@ const std::unordered_map Status::statusMessageMap = { {StatusCode::HF_GIT_CLONE_FAILED, "Failed in libgit2 execution of clone method"}, {StatusCode::HF_GIT_STATUS_FAILED, "Failed in libgit2 execution of status method"}, {StatusCode::HF_GIT_STATUS_UNCLEAN, "Unclean status detected in libgit2 cloned repository"}, - {StatusCode::PARTIAL_END, "Request has finished and no further communication is needed"}, {StatusCode::NONEXISTENT_PATH, "Nonexistent path"}, diff --git a/src/test/pull_hf_model_test.cpp b/src/test/pull_hf_model_test.cpp index 9eba14b103..6043ea380e 100644 --- a/src/test/pull_hf_model_test.cpp +++ b/src/test/pull_hf_model_test.cpp @@ -153,7 +153,7 @@ const std::string expectedGraphContentsDraft = R"( )"; TEST_F(HfDownloaderPullHfModel, PositiveDownload) { - // GTEST_SKIP() << "Skipping test in CI - PositiveDownloadAndStart has full scope testing."; + GTEST_SKIP() << "Skipping test in CI - PositiveDownloadAndStart has full scope testing."; std::string modelName = "OpenVINO/Phi-3-mini-FastDraft-50M-int8-ov"; std::string downloadPath = ovms::FileSystem::joinPath({this->directoryPath, "repository"}); std::string task = "text_generation"; @@ -178,14 +178,16 @@ bool removeSecondHalf(const std::string& filrStr) { ec.clear(); if (!fs::exists(file, ec) || !fs::is_regular_file(file, ec)) { - if (!ec) ec = std::make_error_code(std::errc::no_such_file_or_directory); + if (!ec) + ec = std::make_error_code(std::errc::no_such_file_or_directory); return false; } const std::uintmax_t size = fs::file_size(file, ec); - if (ec) return false; + if (ec) + return false; - const std::uintmax_t newSize = size / 2; // floor(size/2) + const std::uintmax_t newSize = size / 2; // floor(size/2) fs::resize_file(file, newSize, ec); return !ec; } @@ -196,10 +198,9 @@ bool createGitLfsPointerFile(const std::string& path) { return false; } - file << - "version https://git-lfs.github.com/spec/v1\n" - "oid sha256:cecf0224201415144c00cf3a6cf3350306f9c78888d631eb590939a63722fefa\n" - "size 52417240\n"; + file << "version https://git-lfs.github.com/spec/v1\n" + "oid sha256:cecf0224201415144c00cf3a6cf3350306f9c78888d631eb590939a63722fefa\n" + "size 52417240\n"; return true; } @@ -221,7 +222,7 @@ std::string sha256File(std::string_view path, std::error_code& ec) { } // Read in chunks to support large files without high memory usage. - std::vector buffer(1 << 20); // 1 MiB + std::vector buffer(1 << 20); // 1 MiB while (ifs) { ifs.read(reinterpret_cast(buffer.data()), static_cast(buffer.size())); std::streamsize got = ifs.gcount(); @@ -232,7 +233,7 @@ std::string sha256File(std::string_view path, std::error_code& ec) { } } } - if (!ifs.eof()) { // read failed not due to EOF + if (!ifs.eof()) { // read failed not due to EOF ec = std::make_error_code(std::errc::io_error); return {}; } From 17fcd6122fe52e48a39f36a8acfde41a88e4938b Mon Sep 17 00:00:00 2001 From: Rafal Sapala Date: Wed, 11 Mar 2026 13:10:42 +0100 Subject: [PATCH 28/33] Use VS cmake --- windows_test.bat | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/windows_test.bat b/windows_test.bat index 2fd5dcbfa7..f1a5a27b05 100644 --- a/windows_test.bat +++ b/windows_test.bat @@ -47,7 +47,7 @@ set "runTest=%cd%\bazel-bin\src\ovms_test.exe --gtest_filter=!gtestFilter! > win :: Setting PATH environment variable based on default windows node settings: Added ovms_windows specific python settings and c:/opt and removed unused Nvidia and OCL specific tools. :: When changing the values here you can print the node default PATH value and base your changes on it. -set "setPath=C:\opt;C:\opt\Python312\;C:\opt\Python312\Scripts\;C:\opt\msys64\usr\bin\;C:\opt\curl-8.18.0_4-win64-mingw\bin;%PATH%;" +set "setPath=C:\opt;C:\opt\Python312\;C:\opt\Python312\Scripts\;C:\opt\msys64\usr\bin\;C:\opt\curl-8.18.0_4-win64-mingw\bin;c:\Program Files (x86)\Microsoft Visual Studio\2022\BuildTools\Common7\IDE\CommonExtensions\Microsoft\CMake\CMake\bin\;%PATH%;" set "setPythonPath=%cd%\bazel-out\x64_windows-opt\bin\src\python\binding" set "BAZEL_SH=C:\opt\msys64\usr\bin\bash.exe" From 10419500d2ec940b7d944cedef18a98e39f640fa Mon Sep 17 00:00:00 2001 From: Rafal Sapala Date: Wed, 11 Mar 2026 16:21:33 +0100 Subject: [PATCH 29/33] Short name --- src/test/pull_hf_model_test.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/pull_hf_model_test.cpp b/src/test/pull_hf_model_test.cpp index 6043ea380e..0290e6bf25 100644 --- a/src/test/pull_hf_model_test.cpp +++ b/src/test/pull_hf_model_test.cpp @@ -253,7 +253,7 @@ std::string sha256File(std::string_view path, std::error_code& ec) { return oss.str(); } -TEST_F(HfDownloaderPullHfModel, PositiveDownloadAndResumeFromPArtialDownload) { +TEST_F(HfDownloaderPullHfModel, Resume) { std::string modelName = "OpenVINO/Phi-3-mini-FastDraft-50M-int8-ov"; std::string downloadPath = ovms::FileSystem::joinPath({this->directoryPath, "repository"}); std::string task = "text_generation"; From 5a1d74051363931f4ebd6a3c4c81b7b98977b5d0 Mon Sep 17 00:00:00 2001 From: Rafal Sapala Date: Wed, 11 Mar 2026 17:19:42 +0100 Subject: [PATCH 30/33] Unit tests --- src/BUILD | 1 + src/pull_module/libgit2.cpp | 14 +- src/pull_module/libgit2.hpp | 12 +- src/test/libgit2_test.cpp | 334 ++++++++++++++++++++++++++++++++++++ 4 files changed, 353 insertions(+), 8 deletions(-) create mode 100644 src/test/libgit2_test.cpp diff --git a/src/BUILD b/src/BUILD index d3e5af3861..b9fb30e69b 100644 --- a/src/BUILD +++ b/src/BUILD @@ -2470,6 +2470,7 @@ cc_test( "test/kfs_rest_test.cpp", "test/kfs_rest_parser_test.cpp", "test/layout_test.cpp", + "test/libgit2_test.cpp", "test/metric_config_test.cpp", "test/metrics_test.cpp", "test/metrics_flow_test.cpp", diff --git a/src/pull_module/libgit2.cpp b/src/pull_module/libgit2.cpp index 14f7bcdb89..9b9b29e1f0 100644 --- a/src/pull_module/libgit2.cpp +++ b/src/pull_module/libgit2.cpp @@ -312,7 +312,7 @@ Status HfDownloader::CheckRepositoryStatus(bool checkUntracked) { } while (0) // Trim trailing '\r' (for CRLF files) and surrounding spaces -static inline void rtrimCrLfWhitespace(std::string& s) { +void rtrimCrLfWhitespace(std::string& s) { if (!s.empty() && s.back() == '\r') s.pop_back(); // remove trailing '\r' while (!s.empty() && std::isspace(static_cast(s.back()))) @@ -325,7 +325,7 @@ static inline void rtrimCrLfWhitespace(std::string& s) { } // Case-insensitive substring search: returns true if 'needle' is found in 'hay' -static bool containsCaseInsensitive(const std::string& hay, const std::string& needle) { +bool containsCaseInsensitive(const std::string& hay, const std::string& needle) { auto toLower = [](std::string v) { std::transform(v.begin(), v.end(), v.begin(), [](unsigned char c) { return static_cast(std::tolower(c)); }); @@ -338,7 +338,7 @@ static bool containsCaseInsensitive(const std::string& hay, const std::string& n // Read at most the first 3 lines of a file, with a per-line cap to avoid huge reads. // Returns true if successful (even if <3 lines exist; vector will just be shorter). -static bool readFirstThreeLines(const fs::path& p, std::vector& outLines) { +bool readFirstThreeLines(const fs::path& p, std::vector& outLines) { outLines.clear(); std::ifstream in(p, std::ios::in | std::ios::binary); if (!in) @@ -383,7 +383,7 @@ static bool readFirstThreeLines(const fs::path& p, std::vector& out // Check if the first 3 lines contain required keywords in positional order: // line1 -> "version", line2 -> "oid", line3 -> "size" (case-insensitive). -static bool fileHasLfsKeywordsFirst3Positional(const fs::path& p) { +bool fileHasLfsKeywordsFirst3Positional(const fs::path& p) { std::error_code ec; if (!fs::is_regular_file(p, ec)) return false; @@ -401,7 +401,7 @@ static bool fileHasLfsKeywordsFirst3Positional(const fs::path& p) { } // Helper: make path relative to base (best-effort, non-throwing). -static fs::path makeRelativeToBase(const fs::path& path, const fs::path& base) { +fs::path makeRelativeToBase(const fs::path& path, const fs::path& base) { std::error_code ec; // Try fs::relative first (handles canonical comparisons, may fail if on different roots) fs::path rel = fs::relative(path, base, ec); @@ -419,8 +419,8 @@ static fs::path makeRelativeToBase(const fs::path& path, const fs::path& base) { return path; } -// Find all files under 'directory' that satisfy the first-3-lines LFS keyword check. -static std::vector findLfsLikeFiles(const std::string& directory, bool recursive = true) { +// Find all files under 'directory' that satisfy the first-3-lines LFS keyword check. Default: bool recursive = true +std::vector findLfsLikeFiles(const std::string& directory, bool recursive) { std::vector matches; std::error_code ec; diff --git a/src/pull_module/libgit2.hpp b/src/pull_module/libgit2.hpp index fb5549b16d..f2da9f875f 100644 --- a/src/pull_module/libgit2.hpp +++ b/src/pull_module/libgit2.hpp @@ -15,8 +15,10 @@ // limitations under the License. //***************************************************************************** #pragma once -#include +#include #include +#include +#include #include #include @@ -31,6 +33,7 @@ namespace ovms { class Status; +namespace fs = std::filesystem; /* * libgit2 options. 0 is the default value @@ -64,4 +67,11 @@ class HfDownloader : public IModelDownloader { Status RemoveReadonlyFileAttributeFromDir(const std::string& directoryPath); Status CheckRepositoryStatus(bool checkUntracked); }; + +void rtrimCrLfWhitespace(std::string& s); +bool containsCaseInsensitive(const std::string& hay, const std::string& needle); +bool readFirstThreeLines(const fs::path& p, std::vector& outLines); +bool fileHasLfsKeywordsFirst3Positional(const fs::path& p); +fs::path makeRelativeToBase(const fs::path& path, const fs::path& base); +std::vector findLfsLikeFiles(const std::string& directory, bool recursive = true); } // namespace ovms diff --git a/src/test/libgit2_test.cpp b/src/test/libgit2_test.cpp new file mode 100644 index 0000000000..0a2e1f6328 --- /dev/null +++ b/src/test/libgit2_test.cpp @@ -0,0 +1,334 @@ +//***************************************************************************** +// Copyright 2026 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#include +#include +#include +#include + +#include +#include + +#include "src/pull_module/libgit2.hpp" + +#include "environment.hpp" + +namespace fs = std::filesystem; + +TEST(LibGit2RtrimCrLfWhitespace, EmptyString) { + std::string s; + ovms::rtrimCrLfWhitespace(s); + EXPECT_TRUE(s.empty()); +} + +TEST(LibGit2RtrimCrLfWhitespace, NoWhitespace) { + std::string s = "abc"; + ovms::rtrimCrLfWhitespace(s); + EXPECT_EQ(s, "abc"); +} + +TEST(LibGit2RtrimCrLfWhitespace, OnlySpaces) { + std::string s = " "; + ovms::rtrimCrLfWhitespace(s); + EXPECT_EQ(s, ""); +} + +TEST(LibGit2RtrimCrLfWhitespace, LeadingSpacesOnly) { + std::string s = " abc"; + ovms::rtrimCrLfWhitespace(s); + EXPECT_EQ(s, "abc"); +} + +TEST(LibGit2RtrimCrLfWhitespace, TrailingSpacesOnly) { + std::string s = "abc "; + ovms::rtrimCrLfWhitespace(s); + EXPECT_EQ(s, "abc"); +} + +TEST(LibGit2RtrimCrLfWhitespace, LeadingAndTrailingSpaces) { + std::string s = " abc "; + ovms::rtrimCrLfWhitespace(s); + EXPECT_EQ(s, "abc"); +} + +TEST(LibGit2RtrimCrLfWhitespace, TabsAndNewlinesAround) { + std::string s = "\t\n abc \n\t"; + ovms::rtrimCrLfWhitespace(s); + EXPECT_EQ(s, "abc"); +} + +TEST(LibGit2RtrimCrLfWhitespace, AllCWhitespaceAround) { + // Include space, tab, newline, vertical tab, form feed, carriage return + std::string s = " \t\n\v\f\rabc\r\f\v\n\t "; + ovms::rtrimCrLfWhitespace(s); + EXPECT_EQ(s, "abc"); +} + +TEST(LibGit2RtrimCrLfWhitespace, PreserveInternalSpaces) { + std::string s = " a b c "; + ovms::rtrimCrLfWhitespace(s); + EXPECT_EQ(s, "a b c"); +} + +TEST(LibGit2RtrimCrLfWhitespace, TrailingCRLF) { + // Windows-style line ending: "\r\n" + std::string s = "abc\r\n"; + ovms::rtrimCrLfWhitespace(s); + EXPECT_EQ(s, "abc"); +} + +TEST(LibGit2RtrimCrLfWhitespace, TrailingCROnly) { + std::string s = "abc\r"; + ovms::rtrimCrLfWhitespace(s); + EXPECT_EQ(s, "abc"); +} + +TEST(LibGit2RtrimCrLfWhitespace, TrailingLFOnly) { + std::string s = "abc\n"; + ovms::rtrimCrLfWhitespace(s); + EXPECT_EQ(s, "abc"); +} + +TEST(LibGit2RtrimCrLfWhitespace, MultipleTrailingCRs) { + // Only one trailing '\r' is specially removed first, but then trailing + // whitespace loop will remove any remaining CRs (since isspace('\r') == true). + std::string s = "abc\r\r\r"; + ovms::rtrimCrLfWhitespace(s); + EXPECT_EQ(s, "abc"); +} + +TEST(LibGit2RtrimCrLfWhitespace, LeadingCRLFAndSpaces) { + std::string s = "\r\n abc"; + ovms::rtrimCrLfWhitespace(s); + EXPECT_EQ(s, "abc"); +} + +TEST(LibGit2RtrimCrLfWhitespace, InternalCRLFShouldRemainIfNotLeadingOrTrailing) { + // Internal whitespace should be preserved + std::string s = "a\r\nb"; + ovms::rtrimCrLfWhitespace(s); + EXPECT_EQ(s, "a\r\nb"); +} + +TEST(LibGit2RtrimCrLfWhitespace, OnlyCRLFAndWhitespace) { + std::string s = "\r\n\t \r"; + ovms::rtrimCrLfWhitespace(s); + EXPECT_EQ(s, ""); +} + +TEST(LibGit2RtrimCrLfWhitespace, NonAsciiBytesAreNotTrimmedByIsspace) { + // 0xC2 0xA0 is UTF-8 for NO-BREAK SPACE; bytes individually are not ASCII spaces. + // isspace() on unsigned char typically returns false for these bytes in the "C" locale. + // So they should remain unless at edges and recognized by the current locale (usually not). + std::string s = "\xC2""\xA0""abc""\xC2""\xA0"; + ovms::rtrimCrLfWhitespace(s); + // Expect unchanged because these bytes are not recognized by std::isspace in C locale + EXPECT_EQ(s, "\xC2""\xA0""abc""\xC2""\xA0"); +} + +TEST(LibGit2RtrimCrLfWhitespace, Idempotent) { + std::string s = " abc \n"; + ovms::rtrimCrLfWhitespace(s); + auto once = s; + ovms::rtrimCrLfWhitespace(s); + EXPECT_EQ(s, once); +} + + +TEST(LibGit2ContainsCaseInsensitiveTest, ExactMatch) { + EXPECT_TRUE(ovms::containsCaseInsensitive("hello", "hello")); +} + +TEST(LibGit2ContainsCaseInsensitiveTest, MixedCaseMatch) { + EXPECT_TRUE(ovms::containsCaseInsensitive("HeLLo WoRLD", "world")); + EXPECT_TRUE(ovms::containsCaseInsensitive("HeLLo WoRLD", "HELLO")); +} + +TEST(LibGit2ContainsCaseInsensitiveTest, NoMatch) { + EXPECT_FALSE(ovms::containsCaseInsensitive("abcdef", "gh")); +} + +TEST(LibGit2ContainsCaseInsensitiveTest, EmptyNeedleReturnsTrue) { + // Consistent with std::string::find("") → 0 + EXPECT_TRUE(ovms::containsCaseInsensitive("something", "")); +} + +TEST(LibGit2ContainsCaseInsensitiveTest, EmptyHaystackNonEmptyNeedleReturnsFalse) { + EXPECT_FALSE(ovms::containsCaseInsensitive("", "abc")); +} + +TEST(LibGit2ContainsCaseInsensitiveTest, BothEmptyReturnsTrue) { + EXPECT_TRUE(ovms::containsCaseInsensitive("", "")); +} + +TEST(LibGit2ContainsCaseInsensitiveTest, SubstringAtBeginning) { + EXPECT_TRUE(ovms::containsCaseInsensitive("HelloWorld", "hello")); +} + +TEST(LibGit2ContainsCaseInsensitiveTest, SubstringInMiddle) { + EXPECT_TRUE(ovms::containsCaseInsensitive("abcHELLOxyz", "hello")); +} + +TEST(LibGit2ContainsCaseInsensitiveTest, SubstringAtEnd) { + EXPECT_TRUE(ovms::containsCaseInsensitive("testCASE", "case")); +} + +TEST(LibGit2ContainsCaseInsensitiveTest, NoFalsePositives) { + EXPECT_FALSE(ovms::containsCaseInsensitive("aaaaa", "b")); +} + +TEST(LibGit2ContainsCaseInsensitiveTest, UnicodeCharactersSafeButNotSpecialHandled) { + // std::tolower only reliably handles unsigned char range. + // This ensures your implementation does not crash or behave strangely. + EXPECT_FALSE(ovms::containsCaseInsensitive("ĄĆĘŁ", "ę")); // depends on locale; ASCII-only expected false +} + + +// A helper for writing test files. +static fs::path writeTempFile(const std::string& filename, + const std::string& content) { + fs::path p = fs::temp_directory_path() / filename; + std::ofstream out(p, std::ios::binary); + out << content; + return p; +} + +TEST(LibGit2ReadFirstThreeLinesTest, FileNotFoundReturnsFalse) { + std::vector lines; + fs::path p = fs::temp_directory_path() / "nonexistent_12345.txt"; + EXPECT_FALSE(ovms::readFirstThreeLines(p, lines)); + EXPECT_TRUE(lines.empty()); +} + +TEST(LibGit2ReadFirstThreeLinesTest, ReadsExactlyThreeLines) { + fs::path p = writeTempFile("three_lines.txt", + "line1\n" + "line2\n" + "line3\n" + "extra\n"); // should be ignored + + std::vector out; + EXPECT_TRUE(ovms::readFirstThreeLines(p, out)); + ASSERT_EQ(out.size(), 3u); + EXPECT_EQ(out[0], "line1"); + EXPECT_EQ(out[1], "line2"); + EXPECT_EQ(out[2], "line3"); +} + +TEST(LibGit2ReadFirstThreeLinesTest, ReadsFewerThanThreeLines) { + fs::path p = writeTempFile("two_lines.txt", + "alpha\n" + "beta\n"); + + std::vector out; + EXPECT_TRUE(ovms::readFirstThreeLines(p, out)); + ASSERT_EQ(out.size(), 2u); + EXPECT_EQ(out[0], "alpha"); + EXPECT_EQ(out[1], "beta"); +} + +TEST(LibGit2ReadFirstThreeLinesTest, ReadsOneLineOnly) { + fs::path p = writeTempFile("one_line.txt", "solo\n"); + + std::vector out; + EXPECT_TRUE(ovms::readFirstThreeLines(p, out)); + ASSERT_EQ(out.size(), 1u); + EXPECT_EQ(out[0], "solo"); +} + +TEST(LibGit2ReadFirstThreeLinesTest, EmptyFileProducesZeroLinesAndReturnsTrue) { + fs::path p = writeTempFile("empty.txt", ""); + + std::vector out; + EXPECT_TRUE(ovms::readFirstThreeLines(p, out)); + EXPECT_TRUE(out.empty()); +} + +TEST(LibGit2ReadFirstThreeLinesTest, CRLFIsTrimmedCorrectly) { + fs::path p = writeTempFile("crlf.txt", + "hello\r\n" + "world\r\n"); + + std::vector out; + EXPECT_TRUE(ovms::readFirstThreeLines(p, out)); + ASSERT_EQ(out.size(), 2u); + EXPECT_EQ(out[0], "hello"); + EXPECT_EQ(out[1], "world"); +} + +TEST(LibGit2ReadFirstThreeLinesTest, LoneCRAndLFAreTrimmed) { + fs::path p = writeTempFile("mixed_newlines.txt", + "a\r" + "b\n" + "c\r\n"); + + std::vector out; + EXPECT_TRUE(ovms::readFirstThreeLines(p, out)); + + ASSERT_EQ(out.size(), 3u); + EXPECT_EQ(out[0], "a"); + EXPECT_EQ(out[1], "b"); + EXPECT_EQ(out[2], "c"); +} + +TEST(LibGit2ReadFirstThreeLinesTest, VeryLongLineTriggersDrainLogic) { + constexpr size_t kMax = 8192; + std::string longLine(kMax, 'x'); + std::string content = longLine + "OVERFLOWTHATSHOULDBEDISCARDED\n" // should be truncated + "line2\n" + "line3\n"; + + fs::path p = writeTempFile("long_line.txt", content); + + std::vector out; + EXPECT_TRUE(ovms::readFirstThreeLines(p, out)); + + ASSERT_EQ(out.size(), 3u); + + // First line should be exactly kMax chars of 'x' + ASSERT_EQ(out[0].size(), kMax); + EXPECT_EQ(out[0], std::string(kMax, 'x')); + + EXPECT_EQ(out[1], "line2"); + EXPECT_EQ(out[2], "line3"); +} + +TEST(LibGit2ReadFirstThreeLinesTest, HandlesEOFWithoutNewlineAtEnd) { + fs::path p = writeTempFile("eof_no_newline.txt", + "first\n" + "second\n" + "third_without_newline"); + + std::vector out; + EXPECT_TRUE(ovms::readFirstThreeLines(p, out)); + + ASSERT_EQ(out.size(), 3u); + EXPECT_EQ(out[0], "first"); + EXPECT_EQ(out[1], "second"); + EXPECT_EQ(out[2], "third_without_newline"); +} + +TEST(LibGit2ReadFirstThreeLinesTest, TrailingWhitespacePreservedExceptCRLF) { + fs::path p = writeTempFile("spaces.txt", + "abc \n" + "def\t\t\n"); + + std::vector out; + EXPECT_TRUE(ovms::readFirstThreeLines(p, out)); + + ASSERT_EQ(out.size(), 2u); + EXPECT_EQ(out[0], "abc "); // spaces preserved + EXPECT_EQ(out[1], "def\t\t"); // tabs preserved +} From 214161c1f8584335731923562b1fb50e7e45f390 Mon Sep 17 00:00:00 2001 From: Rafal Sapala Date: Thu, 12 Mar 2026 17:15:00 +0100 Subject: [PATCH 31/33] Unit tests2 --- src/pull_module/libgit2.cpp | 69 +++-- src/test/libgit2_test.cpp | 508 +++++++++++++++++++++++++++++++++--- 2 files changed, 505 insertions(+), 72 deletions(-) diff --git a/src/pull_module/libgit2.cpp b/src/pull_module/libgit2.cpp index 9b9b29e1f0..292530264a 100644 --- a/src/pull_module/libgit2.cpp +++ b/src/pull_module/libgit2.cpp @@ -19,6 +19,7 @@ #include #include #include +#include #include #include @@ -338,46 +339,45 @@ bool containsCaseInsensitive(const std::string& hay, const std::string& needle) // Read at most the first 3 lines of a file, with a per-line cap to avoid huge reads. // Returns true if successful (even if <3 lines exist; vector will just be shorter). -bool readFirstThreeLines(const fs::path& p, std::vector& outLines) { - outLines.clear(); - std::ifstream in(p, std::ios::in | std::ios::binary); + +bool readFirstThreeLines(const std::filesystem::path& p, std::vector& out) { + out.clear(); + + std::ifstream in(p, std::ios::binary); if (!in) return false; - constexpr std::streamsize kMaxPerLine = 8192; - std::string line; - line.reserve(static_cast(kMaxPerLine)); - for (int i = 0; i < 3 && in.good(); ++i) { - line.clear(); - std::streamsize count = 0; - char ch; - bool gotNewline = false; - while (count < kMaxPerLine && in.get(ch)) { - if (ch == '\n') { - gotNewline = true; - break; - } - line.push_back(ch); - ++count; - } - // If we hit kMaxPerLine without encountering '\n', drain until newline to resync - if (count == kMaxPerLine && !gotNewline) { - while (in.get(ch)) { - if (ch == '\n') - break; + line.reserve(256); // small optimization + int c; + + while (out.size() < 3 && (c = in.get()) != EOF) { + if (c == '\r') { + // Handle CR or CRLF as one line ending + int next = in.peek(); + if (next == '\n') { + in.get(); // consume '\n' } + // finalize current line + rtrimCrLfWhitespace(line); + out.push_back(std::move(line)); + line.clear(); + } else if (c == '\n') { + // LF line ending + rtrimCrLfWhitespace(line); + out.push_back(std::move(line)); + line.clear(); + } else { + line.push_back(static_cast(c)); } + } - if (!in && line.empty()) { - // EOF with no data accumulated; if previous lines were read, that's fine. - break; - } + // Handle the last line if file did not end with EOL + if (!line.empty() && out.size() < 3) { rtrimCrLfWhitespace(line); - outLines.push_back(line); - if (!in) - break; // Handle EOF gracefully + out.push_back(std::move(line)); } + return true; } @@ -593,12 +593,7 @@ Status HfDownloader::downloadModel() { } SPDLOG_DEBUG("Checking repository status."); - auto status = CheckRepositoryStatus(false); - if (!status.ok()) { - return status; - } - - return StatusCode::OK; + return CheckRepositoryStatus(false); } auto status = IModelDownloader::checkIfOverwriteAndRemove(); diff --git a/src/test/libgit2_test.cpp b/src/test/libgit2_test.cpp index 0a2e1f6328..b221ee9c54 100644 --- a/src/test/libgit2_test.cpp +++ b/src/test/libgit2_test.cpp @@ -15,7 +15,9 @@ //***************************************************************************** #include #include +#include #include +#include #include #include @@ -132,10 +134,18 @@ TEST(LibGit2RtrimCrLfWhitespace, NonAsciiBytesAreNotTrimmedByIsspace) { // 0xC2 0xA0 is UTF-8 for NO-BREAK SPACE; bytes individually are not ASCII spaces. // isspace() on unsigned char typically returns false for these bytes in the "C" locale. // So they should remain unless at edges and recognized by the current locale (usually not). - std::string s = "\xC2""\xA0""abc""\xC2""\xA0"; + std::string s = "\xC2" + "\xA0" + "abc" + "\xC2" + "\xA0"; ovms::rtrimCrLfWhitespace(s); // Expect unchanged because these bytes are not recognized by std::isspace in C locale - EXPECT_EQ(s, "\xC2""\xA0""abc""\xC2""\xA0"); + EXPECT_EQ(s, "\xC2" + "\xA0" + "abc" + "\xC2" + "\xA0"); } TEST(LibGit2RtrimCrLfWhitespace, Idempotent) { @@ -146,7 +156,6 @@ TEST(LibGit2RtrimCrLfWhitespace, Idempotent) { EXPECT_EQ(s, once); } - TEST(LibGit2ContainsCaseInsensitiveTest, ExactMatch) { EXPECT_TRUE(ovms::containsCaseInsensitive("hello", "hello")); } @@ -192,13 +201,12 @@ TEST(LibGit2ContainsCaseInsensitiveTest, NoFalsePositives) { TEST(LibGit2ContainsCaseInsensitiveTest, UnicodeCharactersSafeButNotSpecialHandled) { // std::tolower only reliably handles unsigned char range. // This ensures your implementation does not crash or behave strangely. - EXPECT_FALSE(ovms::containsCaseInsensitive("ĄĆĘŁ", "ę")); // depends on locale; ASCII-only expected false + EXPECT_FALSE(ovms::containsCaseInsensitive("ĄĆĘŁ", "ę")); // depends on locale; ASCII-only expected false } - // A helper for writing test files. static fs::path writeTempFile(const std::string& filename, - const std::string& content) { + const std::string& content) { fs::path p = fs::temp_directory_path() / filename; std::ofstream out(p, std::ios::binary); out << content; @@ -217,7 +225,7 @@ TEST(LibGit2ReadFirstThreeLinesTest, ReadsExactlyThreeLines) { "line1\n" "line2\n" "line3\n" - "extra\n"); // should be ignored + "extra\n"); // should be ignored std::vector out; EXPECT_TRUE(ovms::readFirstThreeLines(p, out)); @@ -283,33 +291,11 @@ TEST(LibGit2ReadFirstThreeLinesTest, LoneCRAndLFAreTrimmed) { EXPECT_EQ(out[2], "c"); } -TEST(LibGit2ReadFirstThreeLinesTest, VeryLongLineTriggersDrainLogic) { - constexpr size_t kMax = 8192; - std::string longLine(kMax, 'x'); - std::string content = longLine + "OVERFLOWTHATSHOULDBEDISCARDED\n" // should be truncated - "line2\n" - "line3\n"; - - fs::path p = writeTempFile("long_line.txt", content); - - std::vector out; - EXPECT_TRUE(ovms::readFirstThreeLines(p, out)); - - ASSERT_EQ(out.size(), 3u); - - // First line should be exactly kMax chars of 'x' - ASSERT_EQ(out[0].size(), kMax); - EXPECT_EQ(out[0], std::string(kMax, 'x')); - - EXPECT_EQ(out[1], "line2"); - EXPECT_EQ(out[2], "line3"); -} - TEST(LibGit2ReadFirstThreeLinesTest, HandlesEOFWithoutNewlineAtEnd) { fs::path p = writeTempFile("eof_no_newline.txt", - "first\n" - "second\n" - "third_without_newline"); + "first\n" + "second\n" + "third_without_newline"); std::vector out; EXPECT_TRUE(ovms::readFirstThreeLines(p, out)); @@ -320,7 +306,7 @@ TEST(LibGit2ReadFirstThreeLinesTest, HandlesEOFWithoutNewlineAtEnd) { EXPECT_EQ(out[2], "third_without_newline"); } -TEST(LibGit2ReadFirstThreeLinesTest, TrailingWhitespacePreservedExceptCRLF) { +TEST(LibGit2ReadFirstThreeLinesTest, TrailingWhitespaceNotPreserved) { fs::path p = writeTempFile("spaces.txt", "abc \n" "def\t\t\n"); @@ -329,6 +315,458 @@ TEST(LibGit2ReadFirstThreeLinesTest, TrailingWhitespacePreservedExceptCRLF) { EXPECT_TRUE(ovms::readFirstThreeLines(p, out)); ASSERT_EQ(out.size(), 2u); - EXPECT_EQ(out[0], "abc "); // spaces preserved - EXPECT_EQ(out[1], "def\t\t"); // tabs preserved + EXPECT_EQ(out[0], "abc"); // spaces preserved + EXPECT_EQ(out[1], "def"); // tabs preserved +} + +// Optional: If you need to call readFirstThreeLines in any test-specific checks, +// declare it too (remove if unused here). +// bool readFirstThreeLines(const fs::path& p, std::vector& out); + +// ---- Test Utilities ---- + +// Create a unique temporary directory inside the system temp directory. +static fs::path createTempDir() { + const fs::path base = fs::temp_directory_path(); + std::random_device rd; + std::mt19937_64 gen(rd()); + std::uniform_int_distribution dist; + + // Try a reasonable number of times to avoid rare collisions + for (int attempt = 0; attempt < 100; ++attempt) { + auto candidate = base / ("lfs_kw_tests_" + std::to_string(dist(gen))); + std::error_code ec; + if (fs::create_directory(candidate, ec)) { + return candidate; + } + // If creation failed due to existing path, loop and try another name + // Otherwise (e.g., permissions), fall through and try again up to limit + } + + throw std::runtime_error("Failed to create a unique temporary directory"); +} + +static fs::path writeFile(const fs::path& dir, const std::string& name, const std::string& content) { + fs::path p = dir / name; + std::ofstream out(p, std::ios::binary); + if (!out) + throw std::runtime_error("Failed to create file: " + p.string()); + out.write(content.data(), static_cast(content.size())); + return p; +} + +// A simple RAII for a temp directory +struct TempDir { + fs::path dir; + TempDir() : + dir(createTempDir()) { + if (dir.empty()) + throw std::runtime_error("Failed to create temp directory"); + } + ~TempDir() { + std::error_code ec; + fs::remove_all(dir, ec); + } +}; + +class LibGit2FileHasLfsKeywordsFirst3PositionalTest : public ::testing::Test { +protected: + TempDir td; +}; + +// ---- Tests ---- + +TEST_F(LibGit2FileHasLfsKeywordsFirst3PositionalTest, ReturnsFalseForNonExistingFile) { + fs::path p = td.dir / "does_not_exist.txt"; + EXPECT_FALSE(ovms::fileHasLfsKeywordsFirst3Positional(p)); +} + +TEST_F(LibGit2FileHasLfsKeywordsFirst3PositionalTest, ReturnsFalseForDirectoryPath) { + // Passing the directory itself (not a regular file) + EXPECT_FALSE(ovms::fileHasLfsKeywordsFirst3Positional(td.dir)); +} + +TEST_F(LibGit2FileHasLfsKeywordsFirst3PositionalTest, ReturnsFalseForEmptyFile) { + auto p = writeFile(td.dir, "empty.txt", ""); + EXPECT_FALSE(ovms::fileHasLfsKeywordsFirst3Positional(p)); +} + +TEST_F(LibGit2FileHasLfsKeywordsFirst3PositionalTest, ReturnsFalseForLessThanThreeLines) { + { + auto p = writeFile(td.dir, "one_line.txt", "version something\n"); + EXPECT_FALSE(ovms::fileHasLfsKeywordsFirst3Positional(p)); + } + { + auto p = writeFile(td.dir, "two_lines.txt", "version x\n" + "oid y\n"); + EXPECT_FALSE(ovms::fileHasLfsKeywordsFirst3Positional(p)); + } +} + +TEST_F(LibGit2FileHasLfsKeywordsFirst3PositionalTest, HappyPathCaseInsensitiveAndExtraContent) { + // Lines contain the keywords somewhere (case-insensitive), extra content is okay. + const std::string content = + " VeRsIoN https://git-lfs.github.com/spec/v1 \n" + "\toid Sha256:abcdef1234567890\n" + "size 999999 \t \n"; + auto p = writeFile(td.dir, "ok.txt", content); + EXPECT_TRUE(ovms::fileHasLfsKeywordsFirst3Positional(p)); +} + +TEST_F(LibGit2FileHasLfsKeywordsFirst3PositionalTest, WrongOrderShouldFail) { + // Put keywords in wrong lines + const std::string content = + "size 100\n" + "version something\n" + "oid abc\n"; + auto p = writeFile(td.dir, "wrong_order.txt", content); + EXPECT_FALSE(ovms::fileHasLfsKeywordsFirst3Positional(p)); +} + +TEST_F(LibGit2FileHasLfsKeywordsFirst3PositionalTest, MissingKeywordShouldFail) { + // Line1 has version, line2 missing oid, line3 has size + const std::string content = + "version v1\n" + "hash sha256:abc\n" + "size 42\n"; + auto p = writeFile(td.dir, "missing_keyword.txt", content); + EXPECT_FALSE(ovms::fileHasLfsKeywordsFirst3Positional(p)); +} + +TEST_F(LibGit2FileHasLfsKeywordsFirst3PositionalTest, MixedNewlines_CR_LF_CRLF_ShouldPass) { + // Requires readFirstThreeLines to treat \r, \n, and \r\n as line breaks. + const std::string content = + "version one\r" + "oid two\n" + "size three\r\n"; + auto p = writeFile(td.dir, "mixed_newlines.txt", content); + EXPECT_TRUE(ovms::fileHasLfsKeywordsFirst3Positional(p)); +} + +TEST_F(LibGit2FileHasLfsKeywordsFirst3PositionalTest, LeadingAndTrailingWhitespaceDoesNotBreak) { + // Assuming readFirstThreeLines trims edge whitespace; otherwise contains() still works + const std::string content = + " version \n" + "\t oid\t\n" + " size \t\n"; + auto p = writeFile(td.dir, "whitespace.txt", content); + EXPECT_TRUE(ovms::fileHasLfsKeywordsFirst3Positional(p)); +} + +TEST_F(LibGit2FileHasLfsKeywordsFirst3PositionalTest, KeywordsMayAppearWithinLongerTextOnEachLine) { + const std::string content = + "prefix-version-suffix\n" + "some_oid_here\n" + "the_size_is_here\n"; + auto p = writeFile(td.dir, "contains_substrings.txt", content); + EXPECT_TRUE(ovms::fileHasLfsKeywordsFirst3Positional(p)); +} + +TEST_F(LibGit2FileHasLfsKeywordsFirst3PositionalTest, CaseInsensitiveCheck) { + const std::string content = + "VerSiOn 1\n" + "OID something\n" + "SiZe 123\n"; + auto p = writeFile(td.dir, "case_insensitive.txt", content); + EXPECT_TRUE(ovms::fileHasLfsKeywordsFirst3Positional(p)); +} + +TEST_F(LibGit2FileHasLfsKeywordsFirst3PositionalTest, ExtraLinesAfterFirstThreeDoNotMatter) { + const std::string content = + "version v1\n" + "oid abc\n" + "size 42\n" + "EXTRA LINE THAT SHOULD NOT AFFECT RESULT\n"; + auto p = writeFile(td.dir, "extra_lines.txt", content); + EXPECT_TRUE(ovms::fileHasLfsKeywordsFirst3Positional(p)); +} + +class LibGit2MakeRelativeToBaseTest : public ::testing::Test { +protected: + TempDir td; +}; + +// Base is an ancestor of path → should return the relative tail. +TEST_F(LibGit2MakeRelativeToBaseTest, BaseIsAncestor) { + fs::path base = td.dir / "root"; + fs::path sub = base / "a" / "b" / "file.txt"; + + std::error_code ec; + fs::create_directories(sub.parent_path(), ec); + + fs::path rel = ovms::makeRelativeToBase(sub, base); + // Expected: "a/b/file.txt" (platform-correct separators) + EXPECT_EQ(rel, fs::path("a") / "b" / "file.txt"); +} + +// Path equals base → fs::relative returns "." (non-empty), we keep it. +TEST_F(LibGit2MakeRelativeToBaseTest, PathEqualsBase) { + fs::path base = td.dir / "same"; + std::error_code ec; + fs::create_directories(base, ec); + + fs::path rel = ovms::makeRelativeToBase(base, base); + EXPECT_EQ(rel, fs::path(".")); +} + +// Sibling subtree: base is ancestor of both; result is still relative path from base. +TEST_F(LibGit2MakeRelativeToBaseTest, SiblingSubtree) { + fs::path base = td.dir / "root2"; + fs::path a = base / "a" / "deep" / "fileA.txt"; + fs::path b = base / "b"; + + std::error_code ec; + fs::create_directories(a.parent_path(), ec); + fs::create_directories(b, ec); + + fs::path rel = ovms::makeRelativeToBase(a, base); + EXPECT_EQ(rel, fs::path("a") / "deep" / "fileA.txt"); +} + +// Base is not an ancestor but on same root → return a proper upward relative like "../x/y". +TEST_F(LibGit2MakeRelativeToBaseTest, BaseIsNotAncestorButSameRoot) { + fs::path base = td.dir / "top" / "left"; + fs::path path = td.dir / "top" / "right" / "x" / "y.txt"; + + std::error_code ec; + fs::create_directories(base, ec); + fs::create_directories(path.parent_path(), ec); + + fs::path rel = ovms::makeRelativeToBase(path, base); + // From .../top/left to .../top/right/x/y.txt → "../right/x/y.txt" + EXPECT_EQ(rel, fs::path("..") / "right" / "x" / "y.txt"); +} + +// Works even if paths do not exist (lexical computation should still yield a sensible result) +TEST_F(LibGit2MakeRelativeToBaseTest, NonExistingPathsLexicalStillWorks) { + fs::path base = td.dir / "ghost" / "base"; + fs::path path = td.dir / "ghost" / "base" / "sub" / "file.dat"; + // No directories created + + fs::path rel = ovms::makeRelativeToBase(path, base); + EXPECT_EQ(rel, fs::path("sub") / "file.dat"); +} + +// Last resort on Windows: different drive letters → fs::relative fails, +// lexically_relative returns empty → function should return filename only. +#ifdef _WIN32 +TEST_F(LibGit2MakeRelativeToBaseTest, DifferentDrivesReturnsFilenameOnly) { + // NOTE: We don't touch the filesystem; we only test the path logic. + // Choose typical drive letters; test won't fail if the drive doesn't exist + // because we don't access the filesystem in lexically_relative path. + fs::path path = fs::path("D:\\folder\\file.txt"); + fs::path base = fs::path("C:\\another\\base"); + + fs::path rel = ovms::makeRelativeToBase(path, base); + EXPECT_EQ(rel, fs::path("file.txt")); +} +#endif + +// If path has no filename (e.g., it's a root), last resort returns path itself. +// On POSIX, "/" has no filename; on Windows, "C:\\" has no filename either. +TEST_F(LibGit2MakeRelativeToBaseTest, NoFilenameEdgeCaseReturnsPathItself) { + fs::path base = td.dir; // arbitrary +#if defined(_WIN32) + // Construct a path that has no filename: root-name + root-directory + // We can't know the system drive at compile time; use a generic root directory. + // For the test, we simulate a root-only path lexically. + fs::path path = fs::path("C:\\"); // has no filename +#else + fs::path path = fs::path("../.."); // has no filename +#endif + + fs::path rel = ovms::makeRelativeToBase(path, base); + EXPECT_EQ(rel, path); +} + +static void mkdirs(const fs::path& p) { + std::error_code ec; + fs::create_directories(p, ec); +} + +class LibGit2FindLfsLikeFilesTest : public ::testing::Test { +protected: + TempDir td; + + // Utility: sort paths lexicographically for deterministic comparison + static void sortPaths(std::vector& v) { + std::sort(v.begin(), v.end(), [](const fs::path& a, const fs::path& b) { + return a.generic_string() < b.generic_string(); + }); + } +}; + +// --- Tests --- + +TEST_F(LibGit2FindLfsLikeFilesTest, NonExistingDirectoryReturnsEmpty) { + fs::path nonexist = td.dir / "does_not_exist"; + auto matches = ovms::findLfsLikeFiles(nonexist.string(), /*recursive=*/true); + EXPECT_TRUE(matches.empty()); +} + +TEST_F(LibGit2FindLfsLikeFilesTest, EmptyDirectoryReturnsEmpty) { + auto matches = ovms::findLfsLikeFiles(td.dir.string(), /*recursive=*/true); + EXPECT_TRUE(matches.empty()); +} + +TEST_F(LibGit2FindLfsLikeFilesTest, NonRecursiveFindsOnlyTopLevelMatches) { + // Layout: + // td.dir/ + // match_top.txt (should match) + // nomatch_top.txt (should not match) + // sub/ + // match_nested.txt (should match but NOT included in non-recursive) + // Matching condition: lines[0] contains "version", lines[1] contains "oid", lines[2] contains "size" + + // Create top-level files + writeFile(td.dir, "match_top.txt", + "version v1\n" + "oid sha256:abc\n" + "size 123\n"); + + writeFile(td.dir, "nomatch_top.txt", + "version v1\n" + "hash something\n" // missing "oid" on line 2 + "size 123\n"); + + // Create nested directory and file + fs::path sub = td.dir / "sub"; + mkdirs(sub); + writeFile(sub, "match_nested.txt", + " VERSION v1 \n" + "\toid: 123\n" + "size: 42\n"); + + auto matches = ovms::findLfsLikeFiles(td.dir.string(), /*recursive=*/false); + sortPaths(matches); + + std::vector expected = {fs::path("match_top.txt")}; + sortPaths(expected); + + EXPECT_EQ(matches, expected); +} + +TEST_F(LibGit2FindLfsLikeFilesTest, RecursiveFindsNestedMatches) { + // Same layout as previous test but recursive = true; should include nested match as relative path + writeFile(td.dir, "top_match.txt", + "version spec\n" + "oid hash\n" + "size 1\n"); + + fs::path sub = td.dir / "a" / "b"; + mkdirs(sub); + writeFile(sub, "nested_match.txt", + "VeRsIoN\n" + "OID x\n" + "SiZe y\n"); + + // Add a deeper non-match to ensure it is ignored + fs::path deeper = td.dir / "a" / "b" / "c"; + mkdirs(deeper); + writeFile(deeper, "deep_nomatch.txt", + "hello\n" + "world\n" + "!\n"); + + auto matches = ovms::findLfsLikeFiles(td.dir.string(), /*recursive=*/true); + sortPaths(matches); + + std::vector expected = { + fs::path("top_match.txt"), + fs::path("a") / "b" / "nested_match.txt"}; + sortPaths(expected); + + EXPECT_EQ(matches, expected); +} + +TEST_F(LibGit2FindLfsLikeFilesTest, MixedNewlinesInMatchingFilesAreHandled) { + // Requires underlying readFirstThreeLines + fileHasLfsKeywordsFirst3Positional to handle \r, \n, \r\n + writeFile(td.dir, "mixed1.txt", + "version one\r" + "oid two\n" + "size three\r\n"); + + auto matches = ovms::findLfsLikeFiles(td.dir.string(), /*recursive=*/false); + + ASSERT_EQ(matches.size(), 1u); + EXPECT_EQ(matches[0], fs::path("mixed1.txt")); +} + +TEST_F(LibGit2FindLfsLikeFilesTest, WrongOrderOrMissingKeywordsAreNotIncluded) { + writeFile(td.dir, "wrong_order.txt", + "size 1\n" + "version 2\n" + "oid 3\n"); // wrong order → should not match + + writeFile(td.dir, "missing_second.txt", + "version v1\n" + "hash something\n" // missing "oid" + "size 3\n"); + + auto matches = ovms::findLfsLikeFiles(td.dir.string(), /*recursive=*/false); + EXPECT_TRUE(matches.empty()); +} + +TEST_F(LibGit2FindLfsLikeFilesTest, OnlyRegularFilesConsidered) { + // Create a directory with LFS-like name to ensure it isn't treated as a file + fs::path lfsdir = td.dir / "version_oid_size_dir"; + mkdirs(lfsdir); + + // No files → nothing should match + auto matches = ovms::findLfsLikeFiles(td.dir.string(), /*recursive=*/true); + EXPECT_TRUE(matches.empty()); +} + +TEST_F(LibGit2FindLfsLikeFilesTest, ReturnsPathsRelativeToBaseDirectory) { + // Ensure results are made relative to the provided base dir. + writeFile(td.dir, "root_match.txt", + "version v\n" + "oid o\n" + "size s\n"); + fs::path sub = td.dir / "x" / "y"; + mkdirs(sub); + writeFile(sub, "nested_match.txt", + "version v\n" + "oid o\n" + "size s\n"); + + auto matches = ovms::findLfsLikeFiles(td.dir.string(), /*recursive=*/true); + sortPaths(matches); + + std::vector expected = { + fs::path("root_match.txt"), + fs::path("x") / "y" / "nested_match.txt"}; + sortPaths(expected); + + EXPECT_EQ(matches, expected); +} + +TEST_F(LibGit2FindLfsLikeFilesTest, NonRecursiveDoesNotDescendButStillUsesRelativePaths) { + fs::path sub = td.dir / "subdir"; + mkdirs(sub); + + writeFile(td.dir, "toplevel.txt", + "version a\n" + "oid b\n" + "size c\n"); + + writeFile(sub, "nested.txt", + "version a\n" + "oid b\n" + "size c\n"); + + auto matches_nonrec = ovms::findLfsLikeFiles(td.dir.string(), /*recursive=*/false); + auto matches_rec = ovms::findLfsLikeFiles(td.dir.string(), /*recursive=*/true); + + // Non-recursive: only top-level + ASSERT_EQ(matches_nonrec.size(), 1u); + EXPECT_EQ(matches_nonrec[0], fs::path("toplevel.txt")); + + // Recursive: both, relative to base dir + sortPaths(matches_rec); + std::vector expected = { + fs::path("toplevel.txt"), + fs::path("subdir") / "nested.txt"}; + sortPaths(expected); + EXPECT_EQ(matches_rec, expected); } From 78fff0273342844a5a3e419fe958fc3920064b0a Mon Sep 17 00:00:00 2001 From: Rafal Sapala Date: Fri, 13 Mar 2026 14:06:08 +0100 Subject: [PATCH 32/33] More tests --- src/pull_module/hf_pull_model_module.hpp | 4 +- src/pull_module/libgit2.cpp | 20 ++++++-- src/status.cpp | 4 +- src/status.hpp | 2 + src/test/pull_hf_model_test.cpp | 59 ++++++++++++++++++------ 5 files changed, 70 insertions(+), 19 deletions(-) diff --git a/src/pull_module/hf_pull_model_module.hpp b/src/pull_module/hf_pull_model_module.hpp index 2742ac23ca..46b92b8b57 100644 --- a/src/pull_module/hf_pull_model_module.hpp +++ b/src/pull_module/hf_pull_model_module.hpp @@ -21,7 +21,7 @@ #include "../capi_frontend/server_settings.hpp" namespace ovms { - +class Libgt2InitGuard; class HfPullModelModule : public Module { protected: HFSettingsImpl hfSettings; @@ -40,4 +40,6 @@ class HfPullModelModule : public Module { static const std::string GIT_SERVER_TIMEOUT_ENV; static const std::string GIT_SSL_CERT_LOCATIONS_ENV; }; + +std::variant> createGuard(); } // namespace ovms diff --git a/src/pull_module/libgit2.cpp b/src/pull_module/libgit2.cpp index 292530264a..a0bedcc593 100644 --- a/src/pull_module/libgit2.cpp +++ b/src/pull_module/libgit2.cpp @@ -186,14 +186,16 @@ Status HfDownloader::RemoveReadonlyFileAttributeFromDir(const std::string& direc class GitRepositoryGuard { public: git_repository* repo = nullptr; + int git_error_class = 0; GitRepositoryGuard(const std::string& path) { int error = git_repository_open_ext(&repo, path.c_str(), 0, nullptr); if (error < 0) { const git_error* err = git_error_last(); - if (err) + if (err) { SPDLOG_ERROR("Repository open failed: {} {}", err->klass, err->message); - else + git_error_class = err->klass; + } else SPDLOG_ERROR("Repository open failed: {}", error); if (repo) git_repository_free(repo); @@ -233,7 +235,12 @@ class GitRepositoryGuard { Status HfDownloader::CheckRepositoryStatus(bool checkUntracked) { GitRepositoryGuard repoGuard(this->downloadPath); if (!repoGuard.get()) { - return StatusCode::HF_GIT_STATUS_FAILED; + if (repoGuard.git_error_class == 2) + return StatusCode::HF_GIT_STATUS_FAILED_TO_RESOLVE_PATH; + else if (repoGuard.git_error_class == 3) + return StatusCode::HF_GIT_LIGIT2_NOT_INITIALIZED; + else + return StatusCode::HF_GIT_STATUS_FAILED; } // HEAD state info bool is_detached = git_repository_head_detached(repoGuard.get()) == 1; @@ -569,7 +576,12 @@ Status HfDownloader::downloadModel() { GitRepositoryGuard repoGuard(this->downloadPath); if (!repoGuard.get()) { std::cout << "Path already exists on local filesystem. And is not a git repository: " << this->downloadPath << std::endl; - return StatusCode::HF_GIT_STATUS_FAILED; + if (repoGuard.git_error_class == 2) + return StatusCode::HF_GIT_STATUS_FAILED_TO_RESOLVE_PATH; + else if (repoGuard.git_error_class == 3) + return StatusCode::HF_GIT_LIGIT2_NOT_INITIALIZED; + else + return StatusCode::HF_GIT_STATUS_FAILED; } // Set repository url diff --git a/src/status.cpp b/src/status.cpp index a7750498e9..0cc057042b 100644 --- a/src/status.cpp +++ b/src/status.cpp @@ -349,7 +349,9 @@ const std::unordered_map Status::statusMessageMap = { {StatusCode::HF_RUN_CONVERT_TOKENIZER_EXPORT_FAILED, "Failed to run convert-tokenizer export command"}, {StatusCode::HF_GIT_CLONE_FAILED, "Failed in libgit2 execution of clone method"}, {StatusCode::HF_GIT_STATUS_FAILED, "Failed in libgit2 execution of status method"}, - {StatusCode::HF_GIT_STATUS_UNCLEAN, "Unclean status detected in libgit2 cloned repository"}, + {StatusCode::HF_GIT_STATUS_FAILED_TO_RESOLVE_PATH, "Failed in libgit2 to check repository status for a given path"}, + {StatusCode::HF_GIT_LIGIT2_NOT_INITIALIZED, "Libgit2 was not initialized"}, + {StatusCode::HF_GIT_STATUS_UNCLEAN, "Unclean status detected in libgit2 repository path"}, {StatusCode::PARTIAL_END, "Request has finished and no further communication is needed"}, {StatusCode::NONEXISTENT_PATH, "Nonexistent path"}, diff --git a/src/status.hpp b/src/status.hpp index 2f72532cfd..02b42886a5 100644 --- a/src/status.hpp +++ b/src/status.hpp @@ -361,6 +361,8 @@ enum class StatusCode { HF_RUN_CONVERT_TOKENIZER_EXPORT_FAILED, HF_GIT_CLONE_FAILED, HF_GIT_STATUS_FAILED, + HF_GIT_STATUS_FAILED_TO_RESOLVE_PATH, + HF_GIT_LIGIT2_NOT_INITIALIZED, HF_GIT_STATUS_UNCLEAN, PARTIAL_END, diff --git a/src/test/pull_hf_model_test.cpp b/src/test/pull_hf_model_test.cpp index 0290e6bf25..5092f49dbf 100644 --- a/src/test/pull_hf_model_test.cpp +++ b/src/test/pull_hf_model_test.cpp @@ -253,6 +253,20 @@ std::string sha256File(std::string_view path, std::error_code& ec) { return oss.str(); } +class TestHfDownloader : public ovms::HfDownloader { +public: + TestHfDownloader(const std::string& sourceModel, const std::string& downloadPath, const std::string& hfEndpoint, const std::string& hfToken, const std::string& httpProxy, bool overwrite) : + HfDownloader(sourceModel, downloadPath, hfEndpoint, hfToken, httpProxy, overwrite) {} + std::string GetRepoUrl() { return HfDownloader::GetRepoUrl(); } + std::string GetRepositoryUrlWithPassword() { return HfDownloader::GetRepositoryUrlWithPassword(); } + bool CheckIfProxySet() { return HfDownloader::CheckIfProxySet(); } + const std::string& getEndpoint() { return this->hfEndpoint; } + const std::string& getProxy() { return this->httpProxy; } + std::string getGraphDirectory(const std::string& downloadPath, const std::string& sourceModel) { return IModelDownloader::getGraphDirectory(downloadPath, sourceModel); } + std::string getGraphDirectory() { return HfDownloader::getGraphDirectory(); } + ovms::Status CheckRepositoryStatus(bool checkUntracked) { return HfDownloader::CheckRepositoryStatus(checkUntracked); } +}; + TEST_F(HfDownloaderPullHfModel, Resume) { std::string modelName = "OpenVINO/Phi-3-mini-FastDraft-50M-int8-ov"; std::string downloadPath = ovms::FileSystem::joinPath({this->directoryPath, "repository"}); @@ -275,6 +289,12 @@ TEST_F(HfDownloaderPullHfModel, Resume) { ASSERT_EQ(expectedGraphContents, removeVersionString(graphContents)) << graphContents; + // Check status function + std::unique_ptr hfDownloader = std::make_unique(modelName, ovms::IModelDownloader::getGraphDirectory(downloadPath, modelName), "", "", "", false); + + // Fails because we want clean and it has the graph.pbtxt after download + ASSERT_EQ(hfDownloader->CheckRepositoryStatus(true).getCode(), ovms::StatusCode::HF_GIT_STATUS_UNCLEAN); + std::error_code ec; ec.clear(); std::string expectedDigest = sha256File(modelPath, ec); @@ -439,19 +459,6 @@ class TestOptimumDownloader : public ovms::OptimumDownloader { bool checkIfTokenizerFileIsExported() { return ovms::OptimumDownloader::checkIfTokenizerFileIsExported(); } }; -class TestHfDownloader : public ovms::HfDownloader { -public: - TestHfDownloader(const std::string& sourceModel, const std::string& downloadPath, const std::string& hfEndpoint, const std::string& hfToken, const std::string& httpProxy, bool overwrite) : - HfDownloader(sourceModel, downloadPath, hfEndpoint, hfToken, httpProxy, overwrite) {} - std::string GetRepoUrl() { return HfDownloader::GetRepoUrl(); } - std::string GetRepositoryUrlWithPassword() { return HfDownloader::GetRepositoryUrlWithPassword(); } - bool CheckIfProxySet() { return HfDownloader::CheckIfProxySet(); } - const std::string& getEndpoint() { return this->hfEndpoint; } - const std::string& getProxy() { return this->httpProxy; } - std::string getGraphDirectory(const std::string& downloadPath, const std::string& sourceModel) { return IModelDownloader::getGraphDirectory(downloadPath, sourceModel); } - std::string getGraphDirectory() { return HfDownloader::getGraphDirectory(); } -}; - TEST(HfDownloaderClassTest, Methods) { std::string modelName = "model/name"; std::string downloadPath = "/path/to/Download"; @@ -475,6 +482,32 @@ TEST(HfDownloaderClassTest, Methods) { ASSERT_EQ(hfDownloader->getGraphDirectory(), expectedPath); } +TEST(HfDownloaderClassTest, RepositoryStatusCheckErrors) { + std::string modelName = "model/name"; + std::string downloadPath = "/path/to/Download"; + std::string hfEndpoint = "www.new_hf.com/"; + std::string hfToken = "123$$o_O123!AAbb"; + std::string httpProxy = "https://proxy_test1:123"; + std::unique_ptr hfDownloader = std::make_unique(modelName, ovms::IModelDownloader::getGraphDirectory(downloadPath, modelName), hfEndpoint, hfToken, httpProxy, false); + + // Fails without libgit init + ASSERT_EQ(hfDownloader->CheckRepositoryStatus(true).getCode(), ovms::StatusCode::HF_GIT_LIGIT2_NOT_INITIALIZED); + ASSERT_EQ(hfDownloader->CheckRepositoryStatus(false).getCode(), ovms::StatusCode::HF_GIT_LIGIT2_NOT_INITIALIZED); + + auto guardOrError = ovms::createGuard(); + ASSERT_EQ(std::holds_alternative(guardOrError), false); + + // Path does not exist + ASSERT_EQ(hfDownloader->CheckRepositoryStatus(true).getCode(), ovms::StatusCode::HF_GIT_STATUS_FAILED_TO_RESOLVE_PATH); + ASSERT_EQ(hfDownloader->CheckRepositoryStatus(false).getCode(), ovms::StatusCode::HF_GIT_STATUS_FAILED_TO_RESOLVE_PATH); + + // Path not a git repository + downloadPath = getGenericFullPathForSrcTest("/tmp/"); + std::unique_ptr existingHfDownloader = std::make_unique(modelName, downloadPath, hfEndpoint, hfToken, httpProxy, false); + ASSERT_EQ(existingHfDownloader->CheckRepositoryStatus(true).getCode(), ovms::StatusCode::HF_GIT_STATUS_FAILED); + ASSERT_EQ(existingHfDownloader->CheckRepositoryStatus(false).getCode(), ovms::StatusCode::HF_GIT_STATUS_FAILED); +} + class TestOptimumDownloaderSetup : public ::testing::Test { public: ovms::HFSettingsImpl inHfSettings; From abacfa8931cce211657f22154b194dadba7aa43e Mon Sep 17 00:00:00 2001 From: Rafal Sapala Date: Fri, 13 Mar 2026 14:16:53 +0100 Subject: [PATCH 33/33] Style --- src/pull_module/libgit2.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/pull_module/libgit2.cpp b/src/pull_module/libgit2.cpp index a0bedcc593..d2985ded17 100644 --- a/src/pull_module/libgit2.cpp +++ b/src/pull_module/libgit2.cpp @@ -195,8 +195,9 @@ class GitRepositoryGuard { if (err) { SPDLOG_ERROR("Repository open failed: {} {}", err->klass, err->message); git_error_class = err->klass; - } else + } else { SPDLOG_ERROR("Repository open failed: {}", error); + } if (repo) git_repository_free(repo); }