From 111746b7cc1eb16c46dbd03603e0b7696541c7bd Mon Sep 17 00:00:00 2001 From: Kory Draughn Date: Sat, 16 Aug 2025 18:51:27 -0400 Subject: [PATCH 1/4] [418] Implement support for system quotas This commit introduces a new endpoint, /quotas, which supports the following operations: - stat - set_group_quota - recalculate --- API.md | 122 +++++++ CMakeLists.txt | 1 + core/src/main.cpp | 1 + endpoints/CMakeLists.txt | 1 + endpoints/quotas/CMakeLists.txt | 31 ++ endpoints/quotas/src/main.cpp | 319 ++++++++++++++++++ .../irods/private/http_api/handlers.hpp | 2 + test/config.py | 3 +- test/test_irods_http_api.py | 278 +++++++++++++++ 9 files changed, 757 insertions(+), 1 deletion(-) create mode 100644 endpoints/quotas/CMakeLists.txt create mode 100644 endpoints/quotas/src/main.cpp diff --git a/API.md b/API.md index 3c50efda..aa6cf3d4 100644 --- a/API.md +++ b/API.md @@ -1472,6 +1472,128 @@ If an HTTP status code of 200 is returned, the body of the response will contain } ``` +## Quota Operations + +### stat + +Returns global and resource quota information for one or more groups. + +#### Request + +HTTP Method: GET + +```bash +curl http://localhost:/irods-http-api//quotas \ + -H 'Authorization: Bearer ' \ + --data-urlencode 'op=stat' \ + --data-urlencode 'group=' \ # The group which to report quotas for. Optional. + -G +``` + +If a target group is not provided via the `group` parameter, the HTTP API will return quota information for all groups across the zone. + +#### Response + +If an HTTP status code of 200 is returned, the body of the response will contain JSON. Its structure is shown below. + +```js +{ + "irods_response": { + "status_code": 0 + "status_message": "string" // Optional + }, + "global_quotas": [ + { + "group": "string", + "limit": 0, + "over": 0, + "modified_at": "string" + }, + + // Additional entries ... + ], + "resource_quotas": [ + { + "group": "string", + "resource": "string", + "limit": 0, + "over": 0, + "modified_at": "string" + }, + + // Additional entries ... + ] +} +``` + +If there was an error, expect an HTTP status code in either the 4XX or 5XX range. + +### set_group_quota + +Sets the quota for a group, optionally scoped to a single resource. + +#### Request + +HTTP Method: POST + +```bash +curl http://localhost:/irods-http-api//quotas \ + -H 'Authorization: Bearer ' \ + --data-urlencode 'op=set_group_quota' \ + --data-urlencode 'group=' \ # The group to apply the quota to. + --data-urlencode 'resource=' \ # The resource to apply the quota to. Optional. + --data-urlencode 'quota=' # The number of bytes which will serve as the quota limit. +``` + +If a target resource is not provided via the `resource` parameter, the quota will be treated as a global quota. Writing data to one or more resources will count towards the group quota. + +#### Response + +If an HTTP status code of 200 is returned, the body of the response will contain JSON. Its structure is shown below. + +```js +{ + "irods_response": { + "status_code": 0 + "status_message": "string" // Optional + } +} +``` + +If there was an error, expect an HTTP status code in either the 4XX or 5XX range. + +### recalculate + +Calculate or update quota information based on the state of the catalog. + +> [!IMPORTANT] +> iRODS does not automatically update quota information as data changes. This operation is provided to give administrators control over how frequently quotas are updated. + +#### Request + +HTTP Method: POST + +```bash +curl http://localhost:/irods-http-api//quotas \ + -H 'Authorization: Bearer ' \ + --data-urlencode 'op=recalculate' +``` + +#### Response + +If an HTTP status code of 200 is returned, the body of the response will contain JSON. Its structure is shown below. + +```js +{ + "irods_response": { + "status_code": 0 + "status_message": "string" // Optional + } +} +``` + +If there was an error, expect an HTTP status code in either the 4XX or 5XX range. + ## Resource Operations ### create diff --git a/CMakeLists.txt b/CMakeLists.txt index 24bf1898..8fde691b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -163,6 +163,7 @@ target_link_objects( irods_http_api_endpoint_data_objects irods_http_api_endpoint_information irods_http_api_endpoint_query + irods_http_api_endpoint_quotas irods_http_api_endpoint_resources irods_http_api_endpoint_rules irods_http_api_endpoint_tickets diff --git a/core/src/main.cpp b/core/src/main.cpp index bf8e1ea5..1fe9ad1a 100644 --- a/core/src/main.cpp +++ b/core/src/main.cpp @@ -101,6 +101,7 @@ const irods::http::request_handler_map_type req_handlers{ {IRODS_HTTP_API_BASE_URL "/data-objects", irods::http::handler::data_objects}, {IRODS_HTTP_API_BASE_URL "/info", irods::http::handler::information}, {IRODS_HTTP_API_BASE_URL "/query", irods::http::handler::query}, + {IRODS_HTTP_API_BASE_URL "/quotas", irods::http::handler::quotas}, {IRODS_HTTP_API_BASE_URL "/resources", irods::http::handler::resources}, {IRODS_HTTP_API_BASE_URL "/rules", irods::http::handler::rules}, {IRODS_HTTP_API_BASE_URL "/tickets", irods::http::handler::tickets}, diff --git a/endpoints/CMakeLists.txt b/endpoints/CMakeLists.txt index f9173b73..ec0fdbb3 100644 --- a/endpoints/CMakeLists.txt +++ b/endpoints/CMakeLists.txt @@ -6,6 +6,7 @@ add_subdirectory(collections) add_subdirectory(data_objects) add_subdirectory(information) add_subdirectory(query) +add_subdirectory(quotas) add_subdirectory(resources) add_subdirectory(rules) add_subdirectory(tickets) diff --git a/endpoints/quotas/CMakeLists.txt b/endpoints/quotas/CMakeLists.txt new file mode 100644 index 00000000..540b8db0 --- /dev/null +++ b/endpoints/quotas/CMakeLists.txt @@ -0,0 +1,31 @@ +add_library( + irods_http_api_endpoint_quotas + OBJECT + "${CMAKE_CURRENT_SOURCE_DIR}/src/main.cpp" +) + +target_compile_definitions( + irods_http_api_endpoint_quotas + PRIVATE + ${IRODS_COMPILE_DEFINITIONS} + ${IRODS_COMPILE_DEFINITIONS_PRIVATE} +) + +target_link_libraries( + irods_http_api_endpoint_quotas + PRIVATE + irods_client + CURL::libcurl + nlohmann_json::nlohmann_json +) + +target_include_directories( + irods_http_api_endpoint_quotas + PRIVATE + "${IRODS_HTTP_PROJECT_SOURCE_DIR}/core/include" + "${IRODS_HTTP_PROJECT_BINARY_DIR}/core/include" + "${IRODS_HTTP_PROJECT_SOURCE_DIR}/endpoints/shared/include" + "${IRODS_EXTERNALS_FULLPATH_BOOST}/include" +) + +set_target_properties(irods_http_api_endpoint_quotas PROPERTIES EXCLUDE_FROM_ALL TRUE) diff --git a/endpoints/quotas/src/main.cpp b/endpoints/quotas/src/main.cpp new file mode 100644 index 00000000..c8f069a3 --- /dev/null +++ b/endpoints/quotas/src/main.cpp @@ -0,0 +1,319 @@ +#include "irods/private/http_api/handlers.hpp" + +#include "irods/private/http_api/common.hpp" +#include "irods/private/http_api/globals.hpp" +#include "irods/private/http_api/log.hpp" +#include "irods/private/http_api/session.hpp" +#include "irods/private/http_api/version.hpp" + +#include +#include +#include + +#include +#include +#include + +#include + +#include +#include +#include + +// clang-format off +namespace beast = boost::beast; // from +namespace http = beast::http; // from + +namespace logging = irods::http::log; + +using json = nlohmann::json; +// clang-format on + +// NOLINTNEXTLINE(cppcoreguidelines-macro-usage) +#define IRODS_HTTP_API_ENDPOINT_OPERATION_SIGNATURE(name) \ + auto name( \ + irods::http::session_pointer_type _sess_ptr, \ + irods::http::request_type& _req, \ + irods::http::query_arguments_type& _args) \ + ->void + +namespace +{ + // + // Handler function prototypes + // + + IRODS_HTTP_API_ENDPOINT_OPERATION_SIGNATURE(op_stat); + + IRODS_HTTP_API_ENDPOINT_OPERATION_SIGNATURE(op_set_group_quotas); + IRODS_HTTP_API_ENDPOINT_OPERATION_SIGNATURE(op_recalculate); + + // + // Operation to Handler mappings + // + + // clang-format off + const std::unordered_map handlers_for_get{ + {"stat", op_stat} + }; + + const std::unordered_map handlers_for_post{ + {"set_group_quota", op_set_group_quotas}, + {"recalculate", op_recalculate} + }; + // clang-format on +} // anonymous namespace + +namespace irods::http::handler +{ + // NOLINTNEXTLINE(performance-unnecessary-value-param) + IRODS_HTTP_API_ENDPOINT_ENTRY_FUNCTION_SIGNATURE(quotas) + { + // NOLINTNEXTLINE(performance-unnecessary-value-param) + execute_operation(_sess_ptr, _req, handlers_for_get, handlers_for_post); + } // quotas +} // namespace irods::http::handler + +namespace +{ + // + // Operation handler implementations + // + + // NOLINTNEXTLINE(performance-unnecessary-value-param) + IRODS_HTTP_API_ENDPOINT_OPERATION_SIGNATURE(op_stat) + { + auto result = irods::http::resolve_client_identity(_req); + if (result.response) { + return _sess_ptr->send(std::move(*result.response)); + } + + const auto client_info = result.client_info; + + irods::http::globals::background_task( + [fn = __func__, client_info, _sess_ptr, _req = std::move(_req), _args = std::move(_args)] { + logging::info(*_sess_ptr, "{}: client_info.username = [{}]", fn, client_info.username); + + http::response res{http::status::ok, _req.version()}; + res.set(http::field::server, irods::http::version::server_name); + res.set(http::field::content_type, "application/json"); + res.keep_alive(_req.keep_alive()); + + try { + std::vector resource_quotas; + std::vector global_quotas; + + auto conn = irods::get_connection(client_info.username); + + if (const auto iter = _args.find("group"); iter != std::end(_args)) { + static const auto& local_zone = irods::http::globals::configuration() + .at(json::json_pointer{"/irods_client/zone"}) + .get_ref(); + const auto resource_quota_query = fmt::format( + "select QUOTA_USER_NAME, QUOTA_USER_ZONE, QUOTA_RESC_NAME, QUOTA_LIMIT, QUOTA_OVER, " + "QUOTA_MODIFY_TIME where QUOTA_USER_NAME = '{}' and QUOTA_USER_ZONE = '{}' and " + "QUOTA_RESC_ID != '0'", + iter->second, + local_zone); + for (auto&& row : irods::query{static_cast(conn), resource_quota_query}) { + resource_quotas.emplace_back(json{ + {"group", std::move(row[0])}, + {"resource", std::move(row[2])}, + {"limit", std::stoll(row[3])}, + {"over", std::stoll(row[4])}, + {"modified_at", std::move(row[5])}}); + } + + const auto global_quotas_query = fmt::format( + "select QUOTA_USER_NAME, QUOTA_USER_ZONE, QUOTA_LIMIT, QUOTA_OVER, QUOTA_MODIFY_TIME where " + "QUOTA_USER_NAME = '{}' and QUOTA_USER_ZONE = '{}' and QUOTA_RESC_ID = '0'", + iter->second, + local_zone); + for (auto&& row : irods::query{static_cast(conn), global_quotas_query}) { + global_quotas.emplace_back(json{ + {"group", std::move(row[0])}, + {"limit", std::stoll(row[2])}, + {"over", std::stoll(row[3])}, + {"modified_at", std::move(row[4])}}); + } + } + else { + const auto resource_quota_query = + fmt::format("select QUOTA_USER_NAME, QUOTA_USER_ZONE, QUOTA_RESC_NAME, QUOTA_LIMIT, " + "QUOTA_OVER, QUOTA_MODIFY_TIME where QUOTA_RESC_ID != '0'"); + for (auto&& row : irods::query{static_cast(conn), resource_quota_query}) { + resource_quotas.emplace_back(json{ + {"group", std::move(row[0])}, + {"resource", std::move(row[2])}, + {"limit", std::stoll(row[3])}, + {"over", std::stoll(row[4])}, + {"modified_at", std::move(row[5])}}); + } + + const auto global_quotas_query = + fmt::format("select QUOTA_USER_NAME, QUOTA_USER_ZONE, QUOTA_LIMIT, QUOTA_OVER, " + "QUOTA_MODIFY_TIME where QUOTA_RESC_ID = '0'"); + for (auto&& row : irods::query{static_cast(conn), global_quotas_query}) { + global_quotas.emplace_back(json{ + {"group", std::move(row[0])}, + {"limit", std::stoll(row[2])}, + {"over", std::stoll(row[3])}, + {"modified_at", std::move(row[4])}}); + } + } + + // clang-format off + res.body() = json{ + {"irods_response", {{"status_code", 0}}}, + {"resource_quotas", resource_quotas}, + {"global_quotas", global_quotas} + }.dump(); + // clang-format on + } + catch (const irods::exception& e) { + logging::error(*_sess_ptr, "{}: {}", fn, e.client_display_what()); + // clang-format off + res.body() = json{ + {"irods_response", { + {"status_code", e.code()}, + {"status_message", e.client_display_what()} + }} + }.dump(); + // clang-format on + } + catch (const std::exception& e) { + logging::error(*_sess_ptr, "{}: {}", fn, e.what()); + res.result(http::status::internal_server_error); + } + + res.prepare_payload(); + + return _sess_ptr->send(std::move(res)); + }); + } // op_stat + + // NOLINTNEXTLINE(performance-unnecessary-value-param) + IRODS_HTTP_API_ENDPOINT_OPERATION_SIGNATURE(op_set_group_quotas) + { + auto result = irods::http::resolve_client_identity(_req); + if (result.response) { + return _sess_ptr->send(std::move(*result.response)); + } + + const auto client_info = result.client_info; + + irods::http::globals::background_task( + [fn = __func__, client_info, _sess_ptr, _req = std::move(_req), _args = std::move(_args)] { + logging::info(*_sess_ptr, "{}: client_info.username = [{}]", fn, client_info.username); + + http::response res{http::status::ok, _req.version()}; + res.set(http::field::server, irods::http::version::server_name); + res.set(http::field::content_type, "application/json"); + res.keep_alive(_req.keep_alive()); + + try { + const auto group_iter = _args.find("group"); + if (group_iter == std::end(_args)) { + logging::error(*_sess_ptr, "{}: Missing [group] parameter.", fn); + return _sess_ptr->send(irods::http::fail(res, http::status::bad_request)); + } + + const auto quota_iter = _args.find("quota"); + if (group_iter == std::end(_args)) { + logging::error(*_sess_ptr, "{}: Missing [quota] parameter.", fn); + return _sess_ptr->send(irods::http::fail(res, http::status::bad_request)); + } + + GeneralAdminInput input{}; + input.arg0 = "set-quota"; + input.arg1 = "group"; + input.arg2 = group_iter->second.c_str(); + input.arg4 = quota_iter->second.c_str(); + + // Apply the quota as a resource quota if the user set the resource paramter. + // Otherwise, apply it as a quota across all resources. + const auto resource_iter = _args.find("resource"); + if (resource_iter != std::end(_args)) { + input.arg3 = resource_iter->second.c_str(); + } + else { + input.arg3 = "total"; + } + + auto conn = irods::get_connection(client_info.username); + const auto ec = rcGeneralAdmin(static_cast(conn), &input); + + res.body() = json{{"irods_response", {{"status_code", ec}}}}.dump(); + } + catch (const irods::exception& e) { + logging::error(*_sess_ptr, "{}: {}", fn, e.client_display_what()); + // clang-format off + res.body() = json{ + {"irods_response", { + {"status_code", e.code()}, + {"status_message", e.client_display_what()} + }} + }.dump(); + // clang-format on + } + catch (const std::exception& e) { + logging::error(*_sess_ptr, "{}: {}", fn, e.what()); + res.result(http::status::internal_server_error); + } + + res.prepare_payload(); + + return _sess_ptr->send(std::move(res)); + }); + } // op_set_group_quotas + + // NOLINTNEXTLINE(performance-unnecessary-value-param) + IRODS_HTTP_API_ENDPOINT_OPERATION_SIGNATURE(op_recalculate) + { + auto result = irods::http::resolve_client_identity(_req); + if (result.response) { + return _sess_ptr->send(std::move(*result.response)); + } + + const auto client_info = result.client_info; + + irods::http::globals::background_task( + [fn = __func__, client_info, _sess_ptr, _req = std::move(_req), _args = std::move(_args)] { + logging::info(*_sess_ptr, "{}: client_info.username = [{}]", fn, client_info.username); + + http::response res{http::status::ok, _req.version()}; + res.set(http::field::server, irods::http::version::server_name); + res.set(http::field::content_type, "application/json"); + res.keep_alive(_req.keep_alive()); + + try { + GeneralAdminInput input{}; + input.arg0 = "calculate-usage"; + + auto conn = irods::get_connection(client_info.username); + const auto ec = rcGeneralAdmin(static_cast(conn), &input); + + res.body() = json{{"irods_response", {{"status_code", ec}}}}.dump(); + } + catch (const irods::exception& e) { + logging::error(*_sess_ptr, "{}: {}", fn, e.client_display_what()); + // clang-format off + res.body() = json{ + {"irods_response", { + {"status_code", e.code()}, + {"status_message", e.client_display_what()} + }} + }.dump(); + // clang-format on + } + catch (const std::exception& e) { + logging::error(*_sess_ptr, "{}: {}", fn, e.what()); + res.result(http::status::internal_server_error); + } + + res.prepare_payload(); + + return _sess_ptr->send(std::move(res)); + }); + } // op_recalculate +} // anonymous namespace diff --git a/endpoints/shared/include/irods/private/http_api/handlers.hpp b/endpoints/shared/include/irods/private/http_api/handlers.hpp index 06db5e41..cc22498d 100644 --- a/endpoints/shared/include/irods/private/http_api/handlers.hpp +++ b/endpoints/shared/include/irods/private/http_api/handlers.hpp @@ -24,6 +24,8 @@ namespace irods::http::handler IRODS_HTTP_API_ENDPOINT_ENTRY_FUNCTION_SIGNATURE(query); + IRODS_HTTP_API_ENDPOINT_ENTRY_FUNCTION_SIGNATURE(quotas); + IRODS_HTTP_API_ENDPOINT_ENTRY_FUNCTION_SIGNATURE(resources); IRODS_HTTP_API_ENDPOINT_ENTRY_FUNCTION_SIGNATURE(rules); diff --git a/test/config.py b/test/config.py index dc8f2d2b..d750501e 100644 --- a/test/config.py +++ b/test/config.py @@ -25,7 +25,8 @@ 'irods_zone': 'tempZone', 'irods_server_hostname': 'localhost', - 'run_genquery2_tests': True + 'run_genquery2_tests': True, + 'run_quota_tests': False } schema = { diff --git a/test/test_irods_http_api.py b/test/test_irods_http_api.py index 79bec26b..1a30b84c 100644 --- a/test/test_irods_http_api.py +++ b/test/test_irods_http_api.py @@ -4136,6 +4136,284 @@ def test_server_reports_error_when_http_method_is_not_supported(self): def test_server_reports_error_when_op_is_not_supported(self): do_test_server_reports_error_when_op_is_not_supported(self) +class test_quotas_endpoint(unittest.TestCase): + # This test suite requires the iRODS server to be configured with quota monitoring + # enabled. This is accomplished by setting msiSetRescQuotaPolicy("on") in core.re. + + @classmethod + def setUpClass(cls): + setup_class(cls, {'endpoint_name': 'quotas'}) + + @classmethod + def tearDownClass(cls): + tear_down_class(cls) + + def setUp(self): + self.assertFalse(self._class_init_error, 'Class initialization failed. Cannot continue.') + + def test_setting_a_group_quota_for_a_specific_resource(self): + if not config.test_config.get('run_quota_tests', False): + self.skipTest('Quota tests not enabled. Check [run_quota_tests] in test configuration file.') + + rodsuser_headers = {'Authorization': f'Bearer {self.rodsuser_bearer_token}'} + rodsadmin_headers = {'Authorization': f'Bearer {self.rodsadmin_bearer_token}'} + resource = 'demoResc' + + # Set a quota for the public group. This applies to ALL resources. + r = requests.post(self.url_endpoint, headers=rodsadmin_headers, data={ + 'op': 'set_group_quota', + 'group': 'public', + 'resource': resource, + 'quota': 10 + }) + self.logger.debug(r.content) + self.assertEqual(r.status_code, 200) + self.assertEqual(r.json()['irods_response']['status_code'], 0) + + # Show that the quota is active by exceeding the quota. + data_object = f'/{self.zone_name}/home/{self.rodsuser_username}/quotas_for_one_resource.txt' + try: + # Recalculate the quotas. + r = requests.post(self.url_endpoint, headers=rodsadmin_headers, data={ + 'op': 'recalculate' + }) + self.logger.debug(r.content) + self.assertEqual(r.status_code, 200) + self.assertEqual(r.json()['irods_response']['status_code'], 0) + + # Show the quotas. + r = requests.get(self.url_endpoint, headers=rodsadmin_headers, params={ + 'op': 'stat', + 'group': 'public' + }) + self.logger.debug(r.content) + self.assertEqual(r.status_code, 200) + result = r.json() + self.assertEqual(result['irods_response']['status_code'], 0) + self.assertEqual(result['resource_quotas'][0]['limit'], 10) + self.assertEqual(result['resource_quotas'][0]['over'], -10) + + # Create a data object which does not violate the quota limit. + r = requests.post(f'{self.url_base}/data-objects', headers=rodsuser_headers, data={ + 'op': 'write', + 'lpath': data_object, + 'resource': resource, + 'bytes': 'ok' + }) + self.logger.debug(r.content) + self.assertEqual(r.status_code, 200) + self.assertEqual(r.json()['irods_response']['status_code'], 0) + + # Recalculate the quotas. + r = requests.post(self.url_endpoint, headers=rodsadmin_headers, data={ + 'op': 'recalculate' + }) + self.logger.debug(r.content) + self.assertEqual(r.status_code, 200) + self.assertEqual(r.json()['irods_response']['status_code'], 0) + + # Show the quotas. + r = requests.get(self.url_endpoint, headers=rodsadmin_headers, params={ + 'op': 'stat', + 'group': 'public' + }) + self.logger.debug(r.content) + self.assertEqual(r.status_code, 200) + result = r.json() + self.assertEqual(result['irods_response']['status_code'], 0) + self.assertEqual(result['resource_quotas'][0]['limit'], 10) + self.assertEqual(result['resource_quotas'][0]['over'], -8) + + # Overwrite the data object. This puts the quota in violation. Any attempts to + # write to the data object will result in an error. + r = requests.post(f'{self.url_base}/data-objects', headers=rodsuser_headers, data={ + 'op': 'write', + 'lpath': data_object, + 'resource': resource, + 'bytes': 'This puts the quota in violation.' + }) + self.logger.debug(r.content) + self.assertEqual(r.status_code, 200) + self.assertEqual(r.json()['irods_response']['status_code'], 0) + + # Recalculate the quotas. + r = requests.post(self.url_endpoint, headers=rodsadmin_headers, data={ + 'op': 'recalculate' + }) + self.logger.debug(r.content) + self.assertEqual(r.status_code, 200) + self.assertEqual(r.json()['irods_response']['status_code'], 0) + + # Show the data object cannot be written to even if we're truncating it. + r = requests.post(f'{self.url_base}/data-objects', headers=rodsuser_headers, data={ + 'op': 'write', + 'lpath': data_object, + 'resource': 'demoResc', + 'bytes': 'nope' + }) + self.logger.debug(r.content) + self.assertEqual(r.status_code, 200) + self.assertEqual(r.json()['irods_response']['status_code'], irods_error_codes.INVALID_HANDLE) + + finally: + # Remove the data object. + r = requests.post(f'{self.url_base}/data-objects', headers=rodsuser_headers, data={ + 'op': 'remove', + 'lpath': data_object, + 'catalog-only': 0, + 'no-trash': 1 + }) + self.logger.debug(r.content) + + # Recalculate the quotas. + r = requests.post(self.url_endpoint, headers=rodsadmin_headers, data={ + 'op': 'recalculate' + }) + self.logger.debug(r.content) + + # Show the quotas. + r = requests.get(self.url_endpoint, headers=rodsadmin_headers, params={ + 'op': 'stat', + 'group': 'public' + }) + self.logger.debug(r.content) + + # Disable the quota. + r = requests.post(self.url_endpoint, headers=rodsadmin_headers, data={ + 'op': 'set_group_quota', + 'group': 'public', + 'resource': resource, + 'quota': 0 + }) + self.logger.debug(r.content) + + def test_setting_a_group_quota_for_all_resources(self): + if not config.test_config.get('run_quota_tests', False): + self.skipTest('Quota tests not enabled. Check [run_quota_tests] in test configuration file.') + + rodsuser_headers = {'Authorization': f'Bearer {self.rodsuser_bearer_token}'} + rodsadmin_headers = {'Authorization': f'Bearer {self.rodsadmin_bearer_token}'} + + # Set a quota for the public group. This applies to ALL resources. + r = requests.post(self.url_endpoint, headers=rodsadmin_headers, data={ + 'op': 'set_group_quota', + 'group': 'public', + 'quota': 10 + }) + self.logger.debug(r.content) + self.assertEqual(r.status_code, 200) + self.assertEqual(r.json()['irods_response']['status_code'], 0) + + # Show that the quota is active by exceeding the quota. + data_object = f'/{self.zone_name}/home/{self.rodsuser_username}/quotas_for_all.txt' + try: + # Recalculate the quotas. + r = requests.post(self.url_endpoint, headers=rodsadmin_headers, data={ + 'op': 'recalculate' + }) + self.logger.debug(r.content) + self.assertEqual(r.status_code, 200) + self.assertEqual(r.json()['irods_response']['status_code'], 0) + + # Show the quotas. + r = requests.get(self.url_endpoint, headers=rodsadmin_headers, params={ + 'op': 'stat' + }) + self.logger.debug(r.content) + self.assertEqual(r.status_code, 200) + result = r.json() + self.assertEqual(result['irods_response']['status_code'], 0) + self.assertEqual(result['global_quotas'][0]['limit'], 10) + self.assertEqual(result['global_quotas'][0]['over'], -10) + + # Create a data object which does not violate the quota limit. + r = requests.post(f'{self.url_base}/data-objects', headers=rodsuser_headers, data={ + 'op': 'write', + 'lpath': data_object, + 'bytes': 'ok' + }) + self.logger.debug(r.content) + self.assertEqual(r.status_code, 200) + self.assertEqual(r.json()['irods_response']['status_code'], 0) + + # Recalculate the quotas. + r = requests.post(self.url_endpoint, headers=rodsadmin_headers, data={ + 'op': 'recalculate' + }) + self.logger.debug(r.content) + self.assertEqual(r.status_code, 200) + self.assertEqual(r.json()['irods_response']['status_code'], 0) + + # Show the quotas. + r = requests.get(self.url_endpoint, headers=rodsadmin_headers, params={ + 'op': 'stat' + }) + self.logger.debug(r.content) + self.assertEqual(r.status_code, 200) + result = r.json() + self.assertEqual(result['irods_response']['status_code'], 0) + self.assertEqual(result['global_quotas'][0]['limit'], 10) + self.assertEqual(result['global_quotas'][0]['over'], -8) + + # Overwrite the data object. This puts the quota in violation. Any attempts to + # write to the data object will result in an error. + r = requests.post(f'{self.url_base}/data-objects', headers=rodsuser_headers, data={ + 'op': 'write', + 'lpath': data_object, + 'bytes': 'This puts the quota in violation.' + }) + self.logger.debug(r.content) + self.assertEqual(r.status_code, 200) + self.assertEqual(r.json()['irods_response']['status_code'], 0) + + # Recalculate the quotas. + r = requests.post(self.url_endpoint, headers=rodsadmin_headers, data={ + 'op': 'recalculate' + }) + self.logger.debug(r.content) + self.assertEqual(r.status_code, 200) + self.assertEqual(r.json()['irods_response']['status_code'], 0) + + # Show the data object cannot be written to even if we're truncating it. + r = requests.post(f'{self.url_base}/data-objects', headers=rodsuser_headers, data={ + 'op': 'write', + 'lpath': data_object, + 'bytes': 'nope' + }) + self.logger.debug(r.content) + self.assertEqual(r.status_code, 200) + self.assertEqual(r.json()['irods_response']['status_code'], irods_error_codes.INVALID_HANDLE) + + finally: + # Remove the data object. + r = requests.post(f'{self.url_base}/data-objects', headers=rodsuser_headers, data={ + 'op': 'remove', + 'lpath': data_object, + 'catalog-only': 0, + 'no-trash': 1 + }) + self.logger.debug(r.content) + + # Recalculate the quotas. + r = requests.post(self.url_endpoint, headers=rodsadmin_headers, data={ + 'op': 'recalculate' + }) + self.logger.debug(r.content) + + # Show the quotas. + r = requests.get(self.url_endpoint, headers=rodsadmin_headers, params={ + 'op': 'stat' + }) + self.logger.debug(r.content) + + # Disable the quota. + r = requests.post(self.url_endpoint, headers=rodsadmin_headers, data={ + 'op': 'set_group_quota', + 'group': 'public', + 'quota': 0 + }) + self.logger.debug(r.content) + class test_resources_endpoint(unittest.TestCase): @classmethod From 8d6e2ae76daf946aff8e2a7111ad8263d3b32bb1 Mon Sep 17 00:00:00 2001 From: Kory Draughn Date: Tue, 19 Aug 2025 13:14:45 -0400 Subject: [PATCH 2/4] squash. code review comments --- API.md | 8 ++++---- endpoints/quotas/src/main.cpp | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/API.md b/API.md index aa6cf3d4..e9a1ca5c 100644 --- a/API.md +++ b/API.md @@ -1486,7 +1486,7 @@ HTTP Method: GET curl http://localhost:/irods-http-api//quotas \ -H 'Authorization: Bearer ' \ --data-urlencode 'op=stat' \ - --data-urlencode 'group=' \ # The group which to report quotas for. Optional. + --data-urlencode 'group=' \ # The group on which to report. Optional. -G ``` @@ -1540,8 +1540,8 @@ HTTP Method: POST curl http://localhost:/irods-http-api//quotas \ -H 'Authorization: Bearer ' \ --data-urlencode 'op=set_group_quota' \ - --data-urlencode 'group=' \ # The group to apply the quota to. - --data-urlencode 'resource=' \ # The resource to apply the quota to. Optional. + --data-urlencode 'group=' \ # The group to which the new quota applies. + --data-urlencode 'resource=' \ # The resource to which the new quota applies. Optional. --data-urlencode 'quota=' # The number of bytes which will serve as the quota limit. ``` @@ -1567,7 +1567,7 @@ If there was an error, expect an HTTP status code in either the 4XX or 5XX range Calculate or update quota information based on the state of the catalog. > [!IMPORTANT] -> iRODS does not automatically update quota information as data changes. This operation is provided to give administrators control over how frequently quotas are updated. +> iRODS does not automatically update quota information as data changes. This operation is provided to give administrators control over how frequently totals are calculated. #### Request diff --git a/endpoints/quotas/src/main.cpp b/endpoints/quotas/src/main.cpp index c8f069a3..3a19ed00 100644 --- a/endpoints/quotas/src/main.cpp +++ b/endpoints/quotas/src/main.cpp @@ -230,8 +230,8 @@ namespace input.arg2 = group_iter->second.c_str(); input.arg4 = quota_iter->second.c_str(); - // Apply the quota as a resource quota if the user set the resource paramter. - // Otherwise, apply it as a quota across all resources. + // Apply the quota as a resource quota if the user set the resource parameter. + // Otherwise, apply it as a global quota across all resources. const auto resource_iter = _args.find("resource"); if (resource_iter != std::end(_args)) { input.arg3 = resource_iter->second.c_str(); From cb997b5232be6661eead56a7e60363ea9786a6c3 Mon Sep 17 00:00:00 2001 From: Kory Draughn Date: Tue, 19 Aug 2025 14:42:06 -0400 Subject: [PATCH 3/4] squash. code review - TODOs --- test/config.py | 2 ++ test/test_irods_http_api.py | 3 +++ 2 files changed, 5 insertions(+) diff --git a/test/config.py b/test/config.py index d750501e..d7f9c790 100644 --- a/test/config.py +++ b/test/config.py @@ -26,6 +26,8 @@ 'irods_server_hostname': 'localhost', 'run_genquery2_tests': True, + + # TODO(#395): Set to True when this project includes an automated testing environment. 'run_quota_tests': False } diff --git a/test/test_irods_http_api.py b/test/test_irods_http_api.py index 1a30b84c..f1a2f715 100644 --- a/test/test_irods_http_api.py +++ b/test/test_irods_http_api.py @@ -4137,6 +4137,9 @@ def test_server_reports_error_when_op_is_not_supported(self): do_test_server_reports_error_when_op_is_not_supported(self) class test_quotas_endpoint(unittest.TestCase): + # TODO(irods/irods#8624): Update this comment once iRODS moves configuration of + # quotas into the catalog. + # # This test suite requires the iRODS server to be configured with quota monitoring # enabled. This is accomplished by setting msiSetRescQuotaPolicy("on") in core.re. From 8f0d7d63b9b4f922d8675e9f0e8a0a64c5749fee Mon Sep 17 00:00:00 2001 From: Kory Draughn Date: Wed, 1 Oct 2025 16:24:58 -0400 Subject: [PATCH 4/4] squash. formatting, docs, code comments --- API.md | 4 ++++ endpoints/quotas/src/main.cpp | 14 +++++++------- test/test_irods_http_api.py | 2 +- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/API.md b/API.md index e9a1ca5c..39e7d66f 100644 --- a/API.md +++ b/API.md @@ -1532,6 +1532,8 @@ If there was an error, expect an HTTP status code in either the 4XX or 5XX range Sets the quota for a group, optionally scoped to a single resource. +This operation requires rodsadmin level privileges. + #### Request HTTP Method: POST @@ -1569,6 +1571,8 @@ Calculate or update quota information based on the state of the catalog. > [!IMPORTANT] > iRODS does not automatically update quota information as data changes. This operation is provided to give administrators control over how frequently totals are calculated. +This operation requires rodsadmin level privileges. + #### Request HTTP Method: POST diff --git a/endpoints/quotas/src/main.cpp b/endpoints/quotas/src/main.cpp index 3a19ed00..d113147b 100644 --- a/endpoints/quotas/src/main.cpp +++ b/endpoints/quotas/src/main.cpp @@ -231,7 +231,7 @@ namespace input.arg4 = quota_iter->second.c_str(); // Apply the quota as a resource quota if the user set the resource parameter. - // Otherwise, apply it as a global quota across all resources. + // Otherwise, apply it as a global quota across all resources. const auto resource_iter = _args.find("resource"); if (resource_iter != std::end(_args)) { input.arg3 = resource_iter->second.c_str(); @@ -248,12 +248,12 @@ namespace catch (const irods::exception& e) { logging::error(*_sess_ptr, "{}: {}", fn, e.client_display_what()); // clang-format off - res.body() = json{ - {"irods_response", { - {"status_code", e.code()}, - {"status_message", e.client_display_what()} - }} - }.dump(); + res.body() = json{ + {"irods_response", { + {"status_code", e.code()}, + {"status_message", e.client_display_what()} + }} + }.dump(); // clang-format on } catch (const std::exception& e) { diff --git a/test/test_irods_http_api.py b/test/test_irods_http_api.py index f1a2f715..f72db1f4 100644 --- a/test/test_irods_http_api.py +++ b/test/test_irods_http_api.py @@ -4162,7 +4162,7 @@ def test_setting_a_group_quota_for_a_specific_resource(self): rodsadmin_headers = {'Authorization': f'Bearer {self.rodsadmin_bearer_token}'} resource = 'demoResc' - # Set a quota for the public group. This applies to ALL resources. + # Set a quota for the public group. This only applies to the specified resource. r = requests.post(self.url_endpoint, headers=rodsadmin_headers, data={ 'op': 'set_group_quota', 'group': 'public',