From f225516383984f8ab8cd2cc39cab9e4b2a6b93e3 Mon Sep 17 00:00:00 2001 From: "shuizhao.gh" Date: Wed, 20 May 2026 13:25:59 +0800 Subject: [PATCH 1/6] feat(test): add tests/_helpers + responses-based mock infra Introduce a test infrastructure layer so SLS unit tests can run hermetically. - pyproject.toml: pytest config with `testpaths = [tests/unit, tests/e2e]`, an ``e2e`` marker, and DeprecationWarning filter. - setup.py: declare ``responses`` and ``pytest-mock`` under ``extras_require['test']``. - tests/_helpers/env.py: ``require_sls_env()`` reads the canonical ``LOG_TEST_*`` vars and falls back through the legacy aliases (``ALIYUN_LOG_SAMPLE_*`` / ``TEST_*`` / ``SLS_*``); skips the test if any required field is missing. - tests/_helpers/fakes.py: ``make_client`` / ``mock_sls_response`` / ``error_response`` wrappers around the ``responses`` library, mirroring the SLS error-envelope shape from ``aliyun.log.logexception``. - tests/unit/test_logclient_mock.py: three demo tests that prove the HTTP boundary can be stubbed at the requests level for ``get_logs``, ``put_logs`` (signature header + protobuf body) and the error path. Co-Authored-By: Claude Opus 4.7 --- pyproject.toml | 6 ++ setup.py | 2 + tests/_helpers/__init__.py | 0 tests/_helpers/env.py | 102 ++++++++++++++++++++++++ tests/_helpers/fakes.py | 80 +++++++++++++++++++ tests/unit/__init__.py | 0 tests/unit/test_logclient_mock.py | 128 ++++++++++++++++++++++++++++++ 7 files changed, 318 insertions(+) create mode 100644 pyproject.toml create mode 100644 tests/_helpers/__init__.py create mode 100644 tests/_helpers/env.py create mode 100644 tests/_helpers/fakes.py create mode 100644 tests/unit/__init__.py create mode 100644 tests/unit/test_logclient_mock.py diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..221d12f8 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,6 @@ +[tool.pytest.ini_options] +testpaths = ["tests/unit", "tests/e2e"] +markers = [ + "e2e: requires real SLS endpoint (LOG_TEST_* env vars)", +] +filterwarnings = ["ignore::DeprecationWarning"] diff --git a/setup.py b/setup.py index 4b6923f5..dca1fa88 100755 --- a/setup.py +++ b/setup.py @@ -51,6 +51,8 @@ test_requirements = [ 'pytest', + 'pytest-mock', + 'responses', 'lz4', 'virtualenv', 'zstandard' diff --git a/tests/_helpers/__init__.py b/tests/_helpers/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/_helpers/env.py b/tests/_helpers/env.py new file mode 100644 index 00000000..eb0409f1 --- /dev/null +++ b/tests/_helpers/env.py @@ -0,0 +1,102 @@ +# encoding: utf-8 +"""Env-var driven helpers for e2e tests. + +Tests that need a real SLS endpoint should call ``require_sls_env()`` (or use +the ``sls_env`` / ``e2e_client`` fixtures) so that runs without credentials +skip cleanly instead of failing. + +Standardized variable names (preferred): + + LOG_TEST_ENDPOINT + LOG_TEST_ACCESS_KEY_ID + LOG_TEST_ACCESS_KEY_SECRET + LOG_TEST_PROJECT + LOG_TEST_LOGSTORE + +For backward compatibility we also accept (in priority order after LOG_TEST_*): + + ALIYUN_LOG_SAMPLE_ENDPOINT / _ACCESSID / _ACCESSKEY / _PROJECT / _LOGSTORE + TEST_ENDPOINT / TEST_ACCESS_KEY_ID/ TEST_ACCESS_KEY_SECRET / TEST_PROJECT / TEST_LOGSTORE + SLS_ENDPOINT / SLS_AK_ID / SLS_AK_KEY / SLS_PROJECT / SLS_LOGSTORE +""" + +from __future__ import absolute_import + +import os + +import pytest + + +# Each entry: canonical key -> ordered list of env-var names to try. +_ENV_ALIASES = { + "endpoint": [ + "LOG_TEST_ENDPOINT", + "ALIYUN_LOG_SAMPLE_ENDPOINT", + "TEST_ENDPOINT", + "SLS_ENDPOINT", + ], + "access_key_id": [ + "LOG_TEST_ACCESS_KEY_ID", + "ALIYUN_LOG_SAMPLE_ACCESSID", + "TEST_ACCESS_KEY_ID", + "SLS_AK_ID", + ], + "access_key_secret": [ + "LOG_TEST_ACCESS_KEY_SECRET", + "ALIYUN_LOG_SAMPLE_ACCESSKEY", + "TEST_ACCESS_KEY_SECRET", + "SLS_AK_KEY", + ], + "project": [ + "LOG_TEST_PROJECT", + "ALIYUN_LOG_SAMPLE_PROJECT", + "TEST_PROJECT", + "SLS_PROJECT", + ], + "logstore": [ + "LOG_TEST_LOGSTORE", + "ALIYUN_LOG_SAMPLE_LOGSTORE", + "TEST_LOGSTORE", + "SLS_LOGSTORE", + ], +} + + +def _read_env(): + """Return (env_dict, missing_keys).""" + result = {} + missing = [] + for key, names in _ENV_ALIASES.items(): + value = None + for name in names: + v = os.environ.get(name) + if v: + value = v + break + result[key] = value + if not value: + missing.append(key) + return result, missing + + +def require_sls_env(): + """Return SLS env dict or skip the test if any required field is missing. + + :return: dict with keys endpoint, access_key_id, access_key_secret, project, logstore + """ + env, missing = _read_env() + if missing: + pytest.skip( + "SKIP: requires SLS env, missing {0} (set LOG_TEST_* or one of the legacy aliases)".format(missing) + ) + return env + + +def e2e_client(): + """Build a real LogClient from env vars (skips if env not set).""" + # Imported lazily so that unit tests don't require LogClient deps to load + # when they import from this module transitively. + from aliyun.log import LogClient + + env = require_sls_env() + return LogClient(env["endpoint"], env["access_key_id"], env["access_key_secret"]) diff --git a/tests/_helpers/fakes.py b/tests/_helpers/fakes.py new file mode 100644 index 00000000..0e5367f0 --- /dev/null +++ b/tests/_helpers/fakes.py @@ -0,0 +1,80 @@ +# encoding: utf-8 +"""Mock helpers for unit tests that exercise LogClient without real network. + +These wrap the third-party ``responses`` library so test code can stay short +and so SLS-shaped responses (errorCode/errorMessage envelope, x-log-* headers) +can be produced consistently. +""" + +from __future__ import absolute_import + +import json +import re + + +def make_client( + endpoint="cn-mock.example.com", + access_key_id="mock-id", + access_key="mock-key", + project="mock-proj", +): + """Return a LogClient ready to be used inside a ``responses.activate`` context. + + The endpoint never gets contacted; ``responses`` intercepts requests by URL. + """ + from aliyun.log import LogClient + + client = LogClient(endpoint, access_key_id, access_key) + # Stash project on the client for tests that want a default. + client.test_project = project # not used by LogClient itself + return client + + +def error_response(error_code, error_message, request_id="mock-request-id"): + """Return a JSON string matching the SLS error envelope shape. + + Mirrors :class:`aliyun.log.logexception.LogException` so error parsing + code paths can be exercised end-to-end against a mocked response. + """ + return json.dumps({ + "errorCode": error_code, + "errorMessage": error_message, + "requestId": request_id, + }) + + +def mock_sls_response(rsps, method, url_pattern, status=200, body=None, headers=None): + """Register a response on a ``responses`` RequestsMock. + + :param rsps: a ``responses.RequestsMock`` instance (the value yielded by + ``with responses.RequestsMock() as rsps`` or the module itself when + using ``@responses.activate``). + :param method: HTTP method, e.g. ``"GET"`` / ``"POST"``. + :param url_pattern: regex/string URL pattern (passed through to ``responses``). + :param status: HTTP status code to return. + :param body: bytes/str body. Dicts are JSON-encoded. + :param headers: optional response headers; ``x-log-requestid`` is auto-added. + + Always sets ``Content-Type: application/json`` if not provided. + """ + if isinstance(url_pattern, str) and ("*" in url_pattern or "?" in url_pattern): + url_pattern = re.compile(url_pattern) + + if isinstance(body, dict): + body = json.dumps(body) + if body is None: + body = "" + + final_headers = {"x-log-requestid": "mock-request-id"} + if headers: + final_headers.update(headers) + if "Content-Type" not in final_headers: + final_headers["Content-Type"] = "application/json" + + rsps.add( + method=method, + url=url_pattern, + body=body, + status=status, + headers=final_headers, + ) diff --git a/tests/unit/__init__.py b/tests/unit/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/unit/test_logclient_mock.py b/tests/unit/test_logclient_mock.py new file mode 100644 index 00000000..8ed17a0c --- /dev/null +++ b/tests/unit/test_logclient_mock.py @@ -0,0 +1,128 @@ +# encoding: utf-8 +"""Demo unit tests proving the responses-based mock infrastructure works. + +These tests must NOT touch the network. They run as part of `pytest tests/unit` +in CI, with no SLS env vars configured. +""" + +from __future__ import absolute_import + +import json +import re + +import pytest +import responses + +from aliyun.log import GetLogsRequest, LogClient, LogException, LogItem, PutLogsRequest + +from tests._helpers.fakes import error_response, make_client, mock_sls_response + + +@responses.activate +def test_get_logs_returns_parsed(): + """GET /logs response is parsed into a GetLogsResponse with logs.""" + client = make_client(endpoint="cn-mock.example.com", project="mock-proj") + + body = { + "meta": { + "count": 1, + "progress": "Complete", + "processedRows": 1, + "elapsedMillisecond": 1, + "hasSQL": False, + "whereQuery": "", + "aggQuery": "", + "cpuSec": 0, + "cpuCores": 0, + "keys": ["k"], + "terms": [], + "marker": "", + "mode": 0, + "phraseQueryInfo": {"scanAll": False, "beginOffset": 0, "endOffset": 0}, + "shard": -1, + "scanBytes": 0, + "isAccurate": False, + "limited": 0, + "telementryType": "", + }, + "data": [ + {"__time__": "1700000000", "__source__": "src1", "k": "v"}, + ], + } + + mock_sls_response( + responses, + "POST", + re.compile(r"https?://mock-proj\.cn-mock\.example\.com.*?/logstores/store-1/logs.*"), + status=200, + body=body, + ) + + req = GetLogsRequest( + project="mock-proj", + logstore="store-1", + fromTime=1700000000, + toTime=1700000100, + query="*", + ) + resp = client.get_logs(req) + assert resp.get_count() == 1 + log = resp.get_logs()[0] + assert log.contents.get("k") == "v" + assert log.source == "src1" + + +@responses.activate +def test_put_logs_signs_request(): + """PutLogs sends a protobuf body and the SLS auth header is set.""" + client = make_client(endpoint="cn-mock.example.com", project="mock-proj") + + captured = {} + + def request_callback(request): + captured["headers"] = dict(request.headers) + captured["body"] = request.body + return (200, {"x-log-requestid": "mock-request-id"}, "") + + responses.add_callback( + responses.POST, + re.compile(r"https?://mock-proj\.cn-mock\.example\.com.*?/logstores/store-1/shards/lb"), + callback=request_callback, + ) + + item = LogItem(timestamp=1700000000, contents=[("k", "v")]) + client.put_logs(PutLogsRequest("mock-proj", "store-1", "topic", "src", [item])) + + headers = captured["headers"] + # Authorization header is what SLS uses for AuthV1; AuthV4 sets x-acs-content-sha256. + has_auth = "Authorization" in headers or "authorization" in headers + assert has_auth, "expected an Authorization header on the signed request" + # Body is protobuf -- header should declare it. + assert headers.get("Content-Type") == "application/x-protobuf" + # Body must be non-empty bytes. + assert captured["body"] + + +@responses.activate +def test_error_response_raises_logexception(): + """A 400 with the SLS error envelope is converted to LogException with the right errorCode.""" + client = make_client(endpoint="cn-mock.example.com", project="mock-proj") + + mock_sls_response( + responses, + "POST", + re.compile(r"https?://mock-proj\.cn-mock\.example\.com.*?/logstores/store-1/logs.*"), + status=400, + body=error_response("ParameterInvalid", "bad query"), + ) + + req = GetLogsRequest( + project="mock-proj", + logstore="store-1", + fromTime=1700000000, + toTime=1700000100, + query="*", + ) + with pytest.raises(LogException) as excinfo: + client.get_logs(req) + assert excinfo.value.get_error_code() == "ParameterInvalid" From 4fc13ec429aec86a00483b781c163e3783020657 Mon Sep 17 00:00:00 2001 From: "shuizhao.gh" Date: Wed, 20 May 2026 13:45:32 +0800 Subject: [PATCH 2/6] refactor(test): split tests/ into unit/e2e/samples/examples Reorganize the tests/ directory so a hermetic unit suite can run on every push, while integration tests guarded by `LOG_TEST_*` env vars live under their own subtree and skip cleanly when env is missing. Layout: tests/unit/ pytest collected by default, no network access tests/e2e/ marked @pytest.mark.e2e, auto-skipped via conftest tests/samples/ sample.py / sample_consumer.py (not pytest-collected) tests/examples/ consumer_group_examples / export_examples / es_migration tests/_helpers/ env + responses-based fakes (added in prior commit) Notable moves: tests/ut/ -> tests/unit/ tests/ci/build-test/ -> tests/unit/ tests/es_migration/test_*_converter.py / test_index_* -> tests/unit/ tests/integration_test/ -> tests/e2e/ tests/logclient/ -> tests/e2e/logclient/ tests/ingestion_test/ -> tests/e2e/ingestion/ tests/resource_test/ -> tests/e2e/resource/ tests/schedule_sql_test/ -> tests/e2e/schedule_sql/ tests/etl_test/ -> tests/e2e/etl/ tests/test_alert.py -> tests/e2e/test_alert.py tests/compress_test.py -> split: lz4 part to tests/unit/test_compress.py, integration parts to tests/e2e/test_compress_e2e.py tests/sample*.py -> tests/samples/ tests/{consumer_group,export,rebuild_index}_examples/ -> tests/examples/ Each e2e module declares ``pytestmark = pytest.mark.e2e`` and uses ``require_sls_env()`` from tests/_helpers/env.py rather than reading os.environ inline. Env variable names accept the canonical ``LOG_TEST_*`` plus the legacy aliases ``ALIYUN_LOG_SAMPLE_*`` / ``TEST_*`` / ``SLS_*``. CI (.github/workflows/build.yaml) gains a unit-test job that runs ``pytest tests/unit/`` on every push, plus an e2e-test job that runs only on manual workflow_dispatch and consumes ``LOG_TEST_*`` from secrets. TESTING.md documents the conventions; existing build-test job is unchanged. Co-Authored-By: Claude Opus 4.7 --- .github/workflows/build.yaml | 62 ++- .gitignore | 3 + TESTING.md | 113 +++++ tests/conftest.py | 57 +++ tests/e2e/__init__.py | 0 .../data/docker-stdout-config.json | 0 .../data/feitian_1.json | 0 .../data/feitian_2.json | 0 .../data/json_1.json | 0 .../data/json_2.json | 0 .../data/json_3.json | 0 .../data/json_4_docker.json | 0 .../data/mysql-binlog-config.json | 0 .../data/mysql-rawsql-config.json | 0 .../data/nginx-status-config.json | 0 .../data/ngnix_1.json | 0 .../{integration_test => e2e}/data/reg_1.json | 0 .../{integration_test => e2e}/data/reg_2.json | 0 .../{integration_test => e2e}/data/reg_3.json | 0 .../data/reg_4_docker.json | 0 .../{integration_test => e2e}/data/sep_1.json | 0 .../{integration_test => e2e}/data/sep_2.json | 0 .../{integration_test => e2e}/data/sep_3.json | 0 .../data/sep_4_docker.json | 0 .../data/simple_1.json | 0 .../data/simple_2.json | 0 .../data/simple_3.json | 0 .../data/simple_4_docker.json | 0 .../data/syslog_1.json | 0 tests/e2e/etl/__init__.py | 0 tests/{etl_test => e2e/etl}/config1.py | 0 tests/{etl_test => e2e/etl}/config2.py | 0 tests/{etl_test => e2e/etl}/config3.py | 0 tests/{etl_test => e2e/etl}/config4.py | 0 tests/{etl_test => e2e/etl}/data1.txt | 0 tests/{etl_test => e2e/etl}/data1_test1.py | 0 .../etl}/data1_test1_result.txt | 0 tests/{etl_test => e2e/etl}/data1_test2.py | 0 .../etl}/data1_test2_result.txt | 0 tests/{etl_test => e2e/etl}/data1_test3.py | 0 .../etl}/data1_test3_result.txt | 0 tests/{etl_test => e2e/etl}/data2.txt | 0 tests/{etl_test => e2e/etl}/data2_test1.py | 0 .../etl}/data2_test1_result.txt | 0 .../etl}/data2_test1_result_bk.txt | 0 tests/{etl_test => e2e/etl}/data2_test2.py | 0 tests/{etl_test => e2e/etl}/data2_test3.py | 0 tests/{etl_test => e2e/etl}/data3.txt | 0 tests/{etl_test => e2e/etl}/data3_test1.py | 0 .../etl}/data3_test1_result.txt | 0 tests/{etl_test => e2e/etl}/data4.txt | 0 .../etl}/data4_lookup_csv1.txt | 0 tests/{etl_test => e2e/etl}/data4_test1.py | 0 .../etl}/data4_test1_result.txt | 0 tests/{etl_test => e2e/etl}/etl_test.py | 3 + .../etl}/json_data/CVE-2013-0169.json | 0 .../etl}/json_data/simple_data.json | 0 tests/{etl_test => e2e/etl}/sls_config_1.py | 0 tests/{etl_test => e2e/etl}/test_case.py | 46 +- tests/e2e/ingestion/__init__.py | 0 .../ingestion}/elasticsearch_test.py | 3 + .../ingestion}/kafka_test.py | 3 + .../ingestion}/oss_test.py | 3 + tests/e2e/logclient/__init__.py | 0 tests/{ => e2e}/logclient/test_get_logs.py | 3 + tests/{ => e2e}/logclient/test_project.py | 3 + .../logclient/test_resource_group.py | 3 + tests/{ => e2e}/logclient/test_tags.py | 3 + tests/{integration_test => e2e}/logging.conf | 0 tests/e2e/resource/__init__.py | 0 .../resource}/resource_test.py | 3 + tests/e2e/schedule_sql/__init__.py | 0 .../schedule_sql}/schedule_sql_demo_test.py | 3 + tests/{ => e2e}/test_alert.py | 3 + .../test_compress_e2e.py} | 42 +- .../{integration_test => e2e}/test_entity.py | 401 +++++++++--------- .../test_log_handler.py | 5 +- .../test_logtail_config.py | 211 ++++----- tests/examples/README.md | 15 + .../client_worker_with_query.py | 0 .../copy_data_to_logstore.py | 0 .../keyword_monitor.py | 0 .../keyword_monitor_multiple_logstores.py | 0 .../sync_data_to_splunk.py | 0 .../sync_data_to_splunk_multiple_logstores.py | 0 .../sync_data_to_syslog.py | 0 .../consumer_group_examples/syslogclient.py | 0 .../export_examples/export_odps_sink_demo.py | 0 .../export_examples/export_oss_sink_demo.py | 0 .../magic_ext_usage_demo.ipynb | 0 .../save_to_excel_demo.ipynb | 0 .../rebuild_index_example.py | 0 .../test_migration_manager.py | 0 .../test_migration_manager_with_logclient.py | 0 tests/samples/README.md | 15 + tests/{ => samples}/sample.py | 0 tests/{ => samples}/sample2.py | 0 tests/{ => samples}/sample3.py | 0 tests/{ => samples}/sample_consumer.py | 0 tests/{ => samples}/sample_consumer2.py | 0 tests/{ => samples}/sample_data_redundancy.py | 0 tests/{ => samples}/sample_getcontextlogs.py | 0 tests/{ => samples}/sample_ingestion.py | 0 .../{ => samples}/sample_metric_agg_rules.py | 0 tests/{ => samples}/sample_metric_store.py | 0 tests/{ => samples}/sample_object_api.py | 0 tests/{ => samples}/sample_pipeline_config.py | 0 tests/{ => samples}/sample_shipper.py | 0 tests/{ => samples}/sample_sql.py | 0 tests/{ => samples}/sample_v4_sign.py | 0 tests/{ => samples}/user.csv | 0 .../restrict_config_test_data/config1_fail.py | 0 .../restrict_config_test_data/config1_safe.py | 0 .../config2_fail1.py | 0 .../config2_fail2.py | 0 .../config2_fail3.py | 0 .../restrict_config_test_data/config3_fail.py | 0 .../restrict_config_test_data/config3_safe.py | 0 .../restrict_config_test_data/config4_safe.py | 0 .../restrict_config_test_data/config5_safe.py | 0 .../restrict_config_test_data/config6_safe.py | 0 .../restrict_config_test_data/config7_safe.py | 0 tests/unit/test_auth.py | 90 ++++ .../test_cli_config_check.py} | 0 tests/unit/test_compress.py | 11 + .../test_doc_logitem_converter.py | 3 + .../test_doc_logitem_converter_p27.py | 10 + .../test_index_logstore_mappings.py | 0 .../test_mapping_index_converter.py | 0 .../test_proto_use.py => unit/test_proto.py} | 0 tests/{ci/build-test => unit}/test_type.py | 0 tests/{ut => unit}/test_util.py | 0 .../test_util_python3.py} | 0 tests/ut/test_auth.py | 46 -- 134 files changed, 770 insertions(+), 393 deletions(-) create mode 100644 TESTING.md create mode 100644 tests/conftest.py create mode 100644 tests/e2e/__init__.py rename tests/{integration_test => e2e}/data/docker-stdout-config.json (100%) rename tests/{integration_test => e2e}/data/feitian_1.json (100%) rename tests/{integration_test => e2e}/data/feitian_2.json (100%) rename tests/{integration_test => e2e}/data/json_1.json (100%) rename tests/{integration_test => e2e}/data/json_2.json (100%) rename tests/{integration_test => e2e}/data/json_3.json (100%) rename tests/{integration_test => e2e}/data/json_4_docker.json (100%) rename tests/{integration_test => e2e}/data/mysql-binlog-config.json (100%) rename tests/{integration_test => e2e}/data/mysql-rawsql-config.json (100%) rename tests/{integration_test => e2e}/data/nginx-status-config.json (100%) rename tests/{integration_test => e2e}/data/ngnix_1.json (100%) rename tests/{integration_test => e2e}/data/reg_1.json (100%) rename tests/{integration_test => e2e}/data/reg_2.json (100%) rename tests/{integration_test => e2e}/data/reg_3.json (100%) rename tests/{integration_test => e2e}/data/reg_4_docker.json (100%) rename tests/{integration_test => e2e}/data/sep_1.json (100%) rename tests/{integration_test => e2e}/data/sep_2.json (100%) rename tests/{integration_test => e2e}/data/sep_3.json (100%) rename tests/{integration_test => e2e}/data/sep_4_docker.json (100%) rename tests/{integration_test => e2e}/data/simple_1.json (100%) rename tests/{integration_test => e2e}/data/simple_2.json (100%) rename tests/{integration_test => e2e}/data/simple_3.json (100%) rename tests/{integration_test => e2e}/data/simple_4_docker.json (100%) rename tests/{integration_test => e2e}/data/syslog_1.json (100%) create mode 100644 tests/e2e/etl/__init__.py rename tests/{etl_test => e2e/etl}/config1.py (100%) rename tests/{etl_test => e2e/etl}/config2.py (100%) rename tests/{etl_test => e2e/etl}/config3.py (100%) rename tests/{etl_test => e2e/etl}/config4.py (100%) rename tests/{etl_test => e2e/etl}/data1.txt (100%) rename tests/{etl_test => e2e/etl}/data1_test1.py (100%) rename tests/{etl_test => e2e/etl}/data1_test1_result.txt (100%) rename tests/{etl_test => e2e/etl}/data1_test2.py (100%) rename tests/{etl_test => e2e/etl}/data1_test2_result.txt (100%) rename tests/{etl_test => e2e/etl}/data1_test3.py (100%) rename tests/{etl_test => e2e/etl}/data1_test3_result.txt (100%) rename tests/{etl_test => e2e/etl}/data2.txt (100%) rename tests/{etl_test => e2e/etl}/data2_test1.py (100%) rename tests/{etl_test => e2e/etl}/data2_test1_result.txt (100%) rename tests/{etl_test => e2e/etl}/data2_test1_result_bk.txt (100%) rename tests/{etl_test => e2e/etl}/data2_test2.py (100%) rename tests/{etl_test => e2e/etl}/data2_test3.py (100%) rename tests/{etl_test => e2e/etl}/data3.txt (100%) rename tests/{etl_test => e2e/etl}/data3_test1.py (100%) rename tests/{etl_test => e2e/etl}/data3_test1_result.txt (100%) rename tests/{etl_test => e2e/etl}/data4.txt (100%) rename tests/{etl_test => e2e/etl}/data4_lookup_csv1.txt (100%) rename tests/{etl_test => e2e/etl}/data4_test1.py (100%) rename tests/{etl_test => e2e/etl}/data4_test1_result.txt (100%) rename tests/{etl_test => e2e/etl}/etl_test.py (98%) rename tests/{etl_test => e2e/etl}/json_data/CVE-2013-0169.json (100%) rename tests/{etl_test => e2e/etl}/json_data/simple_data.json (100%) rename tests/{etl_test => e2e/etl}/sls_config_1.py (100%) rename tests/{etl_test => e2e/etl}/test_case.py (99%) create mode 100644 tests/e2e/ingestion/__init__.py rename tests/{ingestion_test => e2e/ingestion}/elasticsearch_test.py (98%) rename tests/{ingestion_test => e2e/ingestion}/kafka_test.py (98%) rename tests/{ingestion_test => e2e/ingestion}/oss_test.py (98%) create mode 100644 tests/e2e/logclient/__init__.py rename tests/{ => e2e}/logclient/test_get_logs.py (99%) rename tests/{ => e2e}/logclient/test_project.py (93%) rename tests/{ => e2e}/logclient/test_resource_group.py (96%) rename tests/{ => e2e}/logclient/test_tags.py (97%) rename tests/{integration_test => e2e}/logging.conf (100%) create mode 100644 tests/e2e/resource/__init__.py rename tests/{resource_test => e2e/resource}/resource_test.py (99%) create mode 100644 tests/e2e/schedule_sql/__init__.py rename tests/{schedule_sql_test => e2e/schedule_sql}/schedule_sql_demo_test.py (99%) rename tests/{ => e2e}/test_alert.py (95%) rename tests/{compress_test.py => e2e/test_compress_e2e.py} (70%) rename tests/{integration_test => e2e}/test_entity.py (96%) rename tests/{integration_test => e2e}/test_log_handler.py (98%) rename tests/{integration_test => e2e}/test_logtail_config.py (97%) create mode 100644 tests/examples/README.md rename tests/{ => examples}/consumer_group_examples/client_worker_with_query.py (100%) rename tests/{ => examples}/consumer_group_examples/copy_data_to_logstore.py (100%) rename tests/{ => examples}/consumer_group_examples/keyword_monitor.py (100%) rename tests/{ => examples}/consumer_group_examples/keyword_monitor_multiple_logstores.py (100%) rename tests/{ => examples}/consumer_group_examples/sync_data_to_splunk.py (100%) rename tests/{ => examples}/consumer_group_examples/sync_data_to_splunk_multiple_logstores.py (100%) rename tests/{ => examples}/consumer_group_examples/sync_data_to_syslog.py (100%) rename tests/{ => examples}/consumer_group_examples/syslogclient.py (100%) rename tests/{ => examples}/export_examples/export_odps_sink_demo.py (100%) rename tests/{ => examples}/export_examples/export_oss_sink_demo.py (100%) rename tests/{ => examples}/jupyter_magic_test/magic_ext_usage_demo.ipynb (100%) rename tests/{ => examples}/jupyter_magic_test/save_to_excel_demo.ipynb (100%) rename tests/{ => examples}/rebuild_index_examples/rebuild_index_example.py (100%) rename tests/{es_migration => examples}/test_migration_manager.py (100%) rename tests/{es_migration => examples}/test_migration_manager_with_logclient.py (100%) create mode 100644 tests/samples/README.md rename tests/{ => samples}/sample.py (100%) rename tests/{ => samples}/sample2.py (100%) rename tests/{ => samples}/sample3.py (100%) rename tests/{ => samples}/sample_consumer.py (100%) rename tests/{ => samples}/sample_consumer2.py (100%) rename tests/{ => samples}/sample_data_redundancy.py (100%) rename tests/{ => samples}/sample_getcontextlogs.py (100%) rename tests/{ => samples}/sample_ingestion.py (100%) rename tests/{ => samples}/sample_metric_agg_rules.py (100%) rename tests/{ => samples}/sample_metric_store.py (100%) rename tests/{ => samples}/sample_object_api.py (100%) rename tests/{ => samples}/sample_pipeline_config.py (100%) rename tests/{ => samples}/sample_shipper.py (100%) rename tests/{ => samples}/sample_sql.py (100%) rename tests/{ => samples}/sample_v4_sign.py (100%) rename tests/{ => samples}/user.csv (100%) rename tests/{cli_config_check => unit}/restrict_config_test_data/config1_fail.py (100%) rename tests/{cli_config_check => unit}/restrict_config_test_data/config1_safe.py (100%) rename tests/{cli_config_check => unit}/restrict_config_test_data/config2_fail1.py (100%) rename tests/{cli_config_check => unit}/restrict_config_test_data/config2_fail2.py (100%) rename tests/{cli_config_check => unit}/restrict_config_test_data/config2_fail3.py (100%) rename tests/{cli_config_check => unit}/restrict_config_test_data/config3_fail.py (100%) rename tests/{cli_config_check => unit}/restrict_config_test_data/config3_safe.py (100%) rename tests/{cli_config_check => unit}/restrict_config_test_data/config4_safe.py (100%) rename tests/{cli_config_check => unit}/restrict_config_test_data/config5_safe.py (100%) rename tests/{cli_config_check => unit}/restrict_config_test_data/config6_safe.py (100%) rename tests/{cli_config_check => unit}/restrict_config_test_data/config7_safe.py (100%) create mode 100644 tests/unit/test_auth.py rename tests/{cli_config_check/test_case.py => unit/test_cli_config_check.py} (100%) create mode 100644 tests/unit/test_compress.py rename tests/{es_migration => unit}/test_doc_logitem_converter.py (95%) rename tests/{es_migration => unit}/test_doc_logitem_converter_p27.py (93%) rename tests/{es_migration => unit}/test_index_logstore_mappings.py (100%) rename tests/{es_migration => unit}/test_mapping_index_converter.py (100%) rename tests/{ci/build-test/test_proto_use.py => unit/test_proto.py} (100%) rename tests/{ci/build-test => unit}/test_type.py (100%) rename tests/{ut => unit}/test_util.py (100%) rename tests/{test_util.py => unit/test_util_python3.py} (100%) delete mode 100644 tests/ut/test_auth.py diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 24ba1ab0..a2e07e11 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -5,6 +5,7 @@ on: branches: [ master ] pull_request: branches: [ master ] + workflow_dispatch: permissions: contents: read @@ -21,7 +22,7 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@v2 - + - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v4 with: @@ -40,7 +41,64 @@ jobs: - name: Run build tests run: | python -m pip install pytest - python -m pytest tests/ci/build-test/ + python -m pytest tests/unit/test_proto.py tests/unit/test_type.py -v + + unit-test: + runs-on: ubuntu-latest + + strategy: + fail-fast: false + matrix: + python-version: [3.7, 3.12] + + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + + - name: Install dependencies (with test extras) + run: | + python -V + python -m pip install --upgrade pip setuptools + python -m pip install -e .[test] + + - name: Run unit tests (no network) + run: | + python -m pytest tests/unit/ -v + + e2e-test: + # Manual-only: hits a real SLS endpoint via LOG_TEST_* secrets. + if: github.event_name == 'workflow_dispatch' + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + - name: Set up Python 3.12 + uses: actions/setup-python@v4 + with: + python-version: '3.12' + + - name: Install dependencies (with test extras) + run: | + python -V + python -m pip install --upgrade pip setuptools + python -m pip install -e .[test] + + - name: Run e2e tests + env: + LOG_TEST_ENDPOINT: ${{ secrets.LOG_TEST_ENDPOINT }} + LOG_TEST_ACCESS_KEY_ID: ${{ secrets.LOG_TEST_ACCESS_KEY_ID }} + LOG_TEST_ACCESS_KEY_SECRET: ${{ secrets.LOG_TEST_ACCESS_KEY_SECRET }} + LOG_TEST_PROJECT: ${{ secrets.LOG_TEST_PROJECT }} + LOG_TEST_LOGSTORE: ${{ secrets.LOG_TEST_LOGSTORE }} + run: | + python -m pytest tests/e2e/ -v -m e2e typing: runs-on: ubuntu-latest diff --git a/.gitignore b/.gitignore index 6b8bfbb5..c1cf8bae 100644 --- a/.gitignore +++ b/.gitignore @@ -18,3 +18,6 @@ cscope.out cscope.po.out tags .venv-stubtest/ +.venv-test/ +.pytest_cache/ +__pycache__/ diff --git a/TESTING.md b/TESTING.md new file mode 100644 index 00000000..80c3a430 --- /dev/null +++ b/TESTING.md @@ -0,0 +1,113 @@ +# Testing + +The Python SDK separates fast, hermetic unit tests from end-to-end tests that +require a real SLS endpoint. + +## Layout + +``` +tests/ + _helpers/ shared env + mock infrastructure + unit/ no network, runs in CI on every push + e2e/ hits a real SLS endpoint, gated by env vars + samples/ runnable example scripts (NOT collected by pytest) + examples/ topic-grouped example scripts (NOT collected by pytest) +``` + +`pyproject.toml` sets `testpaths = ["tests/unit", "tests/e2e"]`, so anything +outside those two roots is ignored by `pytest` collection. + +## Running unit tests + +```sh +pip install -e .[test] +pytest tests/unit/ -v +``` + +Unit tests must not touch the network. The repo includes +`tests/unit/test_logclient_mock.py` as a worked example using `responses`. + +## Running e2e tests + +```sh +export LOG_TEST_ENDPOINT=cn-hangzhou.log.aliyuncs.com +export LOG_TEST_ACCESS_KEY_ID=... +export LOG_TEST_ACCESS_KEY_SECRET=... +export LOG_TEST_PROJECT=... +export LOG_TEST_LOGSTORE=... + +pytest tests/e2e/ -v -m e2e +``` + +If any of the required vars are missing, every e2e test is auto-skipped. + +The conftest also reads legacy variable names so older shell setups keep +working: + +| Canonical (preferred) | Legacy aliases (also accepted) | +|-------------------------------|--------------------------------------------------------------------------------------------------| +| `LOG_TEST_ENDPOINT` | `ALIYUN_LOG_SAMPLE_ENDPOINT`, `TEST_ENDPOINT`, `SLS_ENDPOINT` | +| `LOG_TEST_ACCESS_KEY_ID` | `ALIYUN_LOG_SAMPLE_ACCESSID`, `TEST_ACCESS_KEY_ID`, `SLS_AK_ID` | +| `LOG_TEST_ACCESS_KEY_SECRET` | `ALIYUN_LOG_SAMPLE_ACCESSKEY`, `TEST_ACCESS_KEY_SECRET`, `SLS_AK_KEY` | +| `LOG_TEST_PROJECT` | `ALIYUN_LOG_SAMPLE_PROJECT`, `TEST_PROJECT`, `SLS_PROJECT` | +| `LOG_TEST_LOGSTORE` | `ALIYUN_LOG_SAMPLE_LOGSTORE`, `TEST_LOGSTORE`, `SLS_LOGSTORE` | + +## Writing new tests + +### Unit tests with mocked SLS responses + +Use `responses` plus the helpers in `tests/_helpers/fakes.py`: + +```python +import re + +import pytest +import responses + +from aliyun.log import GetLogsRequest, LogException +from tests._helpers.fakes import error_response, make_client, mock_sls_response + + +@responses.activate +def test_my_query(): + client = make_client(endpoint="cn-mock.example.com", project="mock-proj") + mock_sls_response( + responses, + "POST", + re.compile(r"https?://mock-proj\.cn-mock\.example\.com.*?/logstores/store-1/logs.*"), + status=200, + body={"meta": {"count": 0, "progress": "Complete"}, "data": []}, + ) + req = GetLogsRequest("mock-proj", "store-1", 1700000000, 1700000100, query="*") + assert client.get_logs(req).get_count() == 0 + + +@responses.activate +def test_error_path(): + client = make_client() + mock_sls_response( + responses, "POST", + re.compile(r"https?://.*?/logstores/.*"), + status=400, + body=error_response("ParameterInvalid", "bad"), + ) + with pytest.raises(LogException): + client.get_logs(GetLogsRequest("p", "s", 0, 1, "*")) +``` + +### E2E tests against a real endpoint + +```python +import pytest + +pytestmark = pytest.mark.e2e + + +def test_round_trip(e2e_client, sls_env): + # `e2e_client` is an aliyun.log.LogClient built from LOG_TEST_* env vars. + # `sls_env` exposes endpoint/project/logstore/etc. + e2e_client.list_project(size=1) +``` + +The autouse env check skips the whole test cleanly when env vars are missing, +so contributors without SLS credentials can still run `pytest tests/unit/`. diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 00000000..935f267b --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,57 @@ +# encoding: utf-8 +"""Shared pytest fixtures and collection hooks for the test suite.""" + +from __future__ import absolute_import + +import pytest + +from tests._helpers.env import require_sls_env +from tests._helpers.fakes import make_client + + +def pytest_collection_modifyitems(config, items): + """Attach an autouse-style env check to every test marked ``e2e``. + + We rewrite each e2e item's setup so it calls ``require_sls_env()`` before + the test body runs; missing env vars cause the whole test to skip cleanly. + """ + from tests._helpers.env import _read_env + + _, missing = _read_env() + if not missing: + return + + skip_e2e = pytest.mark.skip( + reason="SKIP: requires SLS env, missing {0} (set LOG_TEST_*)".format(missing) + ) + for item in items: + if "e2e" in item.keywords: + item.add_marker(skip_e2e) + + +@pytest.fixture +def sls_env(): + """Return the SLS env dict for an e2e test, or skip if missing.""" + return require_sls_env() + + +@pytest.fixture +def e2e_client(sls_env): + """Return a real LogClient built from env vars, or skip if env missing.""" + from aliyun.log import LogClient + + return LogClient( + sls_env["endpoint"], + sls_env["access_key_id"], + sls_env["access_key_secret"], + ) + + +@pytest.fixture +def mocked_client(): + """Return a LogClient pre-wired for ``responses``-based unit tests. + + Pair with ``responses.activate`` or a ``responses.RequestsMock`` context + manager to stub the HTTP boundary. + """ + return make_client() diff --git a/tests/e2e/__init__.py b/tests/e2e/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/integration_test/data/docker-stdout-config.json b/tests/e2e/data/docker-stdout-config.json similarity index 100% rename from tests/integration_test/data/docker-stdout-config.json rename to tests/e2e/data/docker-stdout-config.json diff --git a/tests/integration_test/data/feitian_1.json b/tests/e2e/data/feitian_1.json similarity index 100% rename from tests/integration_test/data/feitian_1.json rename to tests/e2e/data/feitian_1.json diff --git a/tests/integration_test/data/feitian_2.json b/tests/e2e/data/feitian_2.json similarity index 100% rename from tests/integration_test/data/feitian_2.json rename to tests/e2e/data/feitian_2.json diff --git a/tests/integration_test/data/json_1.json b/tests/e2e/data/json_1.json similarity index 100% rename from tests/integration_test/data/json_1.json rename to tests/e2e/data/json_1.json diff --git a/tests/integration_test/data/json_2.json b/tests/e2e/data/json_2.json similarity index 100% rename from tests/integration_test/data/json_2.json rename to tests/e2e/data/json_2.json diff --git a/tests/integration_test/data/json_3.json b/tests/e2e/data/json_3.json similarity index 100% rename from tests/integration_test/data/json_3.json rename to tests/e2e/data/json_3.json diff --git a/tests/integration_test/data/json_4_docker.json b/tests/e2e/data/json_4_docker.json similarity index 100% rename from tests/integration_test/data/json_4_docker.json rename to tests/e2e/data/json_4_docker.json diff --git a/tests/integration_test/data/mysql-binlog-config.json b/tests/e2e/data/mysql-binlog-config.json similarity index 100% rename from tests/integration_test/data/mysql-binlog-config.json rename to tests/e2e/data/mysql-binlog-config.json diff --git a/tests/integration_test/data/mysql-rawsql-config.json b/tests/e2e/data/mysql-rawsql-config.json similarity index 100% rename from tests/integration_test/data/mysql-rawsql-config.json rename to tests/e2e/data/mysql-rawsql-config.json diff --git a/tests/integration_test/data/nginx-status-config.json b/tests/e2e/data/nginx-status-config.json similarity index 100% rename from tests/integration_test/data/nginx-status-config.json rename to tests/e2e/data/nginx-status-config.json diff --git a/tests/integration_test/data/ngnix_1.json b/tests/e2e/data/ngnix_1.json similarity index 100% rename from tests/integration_test/data/ngnix_1.json rename to tests/e2e/data/ngnix_1.json diff --git a/tests/integration_test/data/reg_1.json b/tests/e2e/data/reg_1.json similarity index 100% rename from tests/integration_test/data/reg_1.json rename to tests/e2e/data/reg_1.json diff --git a/tests/integration_test/data/reg_2.json b/tests/e2e/data/reg_2.json similarity index 100% rename from tests/integration_test/data/reg_2.json rename to tests/e2e/data/reg_2.json diff --git a/tests/integration_test/data/reg_3.json b/tests/e2e/data/reg_3.json similarity index 100% rename from tests/integration_test/data/reg_3.json rename to tests/e2e/data/reg_3.json diff --git a/tests/integration_test/data/reg_4_docker.json b/tests/e2e/data/reg_4_docker.json similarity index 100% rename from tests/integration_test/data/reg_4_docker.json rename to tests/e2e/data/reg_4_docker.json diff --git a/tests/integration_test/data/sep_1.json b/tests/e2e/data/sep_1.json similarity index 100% rename from tests/integration_test/data/sep_1.json rename to tests/e2e/data/sep_1.json diff --git a/tests/integration_test/data/sep_2.json b/tests/e2e/data/sep_2.json similarity index 100% rename from tests/integration_test/data/sep_2.json rename to tests/e2e/data/sep_2.json diff --git a/tests/integration_test/data/sep_3.json b/tests/e2e/data/sep_3.json similarity index 100% rename from tests/integration_test/data/sep_3.json rename to tests/e2e/data/sep_3.json diff --git a/tests/integration_test/data/sep_4_docker.json b/tests/e2e/data/sep_4_docker.json similarity index 100% rename from tests/integration_test/data/sep_4_docker.json rename to tests/e2e/data/sep_4_docker.json diff --git a/tests/integration_test/data/simple_1.json b/tests/e2e/data/simple_1.json similarity index 100% rename from tests/integration_test/data/simple_1.json rename to tests/e2e/data/simple_1.json diff --git a/tests/integration_test/data/simple_2.json b/tests/e2e/data/simple_2.json similarity index 100% rename from tests/integration_test/data/simple_2.json rename to tests/e2e/data/simple_2.json diff --git a/tests/integration_test/data/simple_3.json b/tests/e2e/data/simple_3.json similarity index 100% rename from tests/integration_test/data/simple_3.json rename to tests/e2e/data/simple_3.json diff --git a/tests/integration_test/data/simple_4_docker.json b/tests/e2e/data/simple_4_docker.json similarity index 100% rename from tests/integration_test/data/simple_4_docker.json rename to tests/e2e/data/simple_4_docker.json diff --git a/tests/integration_test/data/syslog_1.json b/tests/e2e/data/syslog_1.json similarity index 100% rename from tests/integration_test/data/syslog_1.json rename to tests/e2e/data/syslog_1.json diff --git a/tests/e2e/etl/__init__.py b/tests/e2e/etl/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/etl_test/config1.py b/tests/e2e/etl/config1.py similarity index 100% rename from tests/etl_test/config1.py rename to tests/e2e/etl/config1.py diff --git a/tests/etl_test/config2.py b/tests/e2e/etl/config2.py similarity index 100% rename from tests/etl_test/config2.py rename to tests/e2e/etl/config2.py diff --git a/tests/etl_test/config3.py b/tests/e2e/etl/config3.py similarity index 100% rename from tests/etl_test/config3.py rename to tests/e2e/etl/config3.py diff --git a/tests/etl_test/config4.py b/tests/e2e/etl/config4.py similarity index 100% rename from tests/etl_test/config4.py rename to tests/e2e/etl/config4.py diff --git a/tests/etl_test/data1.txt b/tests/e2e/etl/data1.txt similarity index 100% rename from tests/etl_test/data1.txt rename to tests/e2e/etl/data1.txt diff --git a/tests/etl_test/data1_test1.py b/tests/e2e/etl/data1_test1.py similarity index 100% rename from tests/etl_test/data1_test1.py rename to tests/e2e/etl/data1_test1.py diff --git a/tests/etl_test/data1_test1_result.txt b/tests/e2e/etl/data1_test1_result.txt similarity index 100% rename from tests/etl_test/data1_test1_result.txt rename to tests/e2e/etl/data1_test1_result.txt diff --git a/tests/etl_test/data1_test2.py b/tests/e2e/etl/data1_test2.py similarity index 100% rename from tests/etl_test/data1_test2.py rename to tests/e2e/etl/data1_test2.py diff --git a/tests/etl_test/data1_test2_result.txt b/tests/e2e/etl/data1_test2_result.txt similarity index 100% rename from tests/etl_test/data1_test2_result.txt rename to tests/e2e/etl/data1_test2_result.txt diff --git a/tests/etl_test/data1_test3.py b/tests/e2e/etl/data1_test3.py similarity index 100% rename from tests/etl_test/data1_test3.py rename to tests/e2e/etl/data1_test3.py diff --git a/tests/etl_test/data1_test3_result.txt b/tests/e2e/etl/data1_test3_result.txt similarity index 100% rename from tests/etl_test/data1_test3_result.txt rename to tests/e2e/etl/data1_test3_result.txt diff --git a/tests/etl_test/data2.txt b/tests/e2e/etl/data2.txt similarity index 100% rename from tests/etl_test/data2.txt rename to tests/e2e/etl/data2.txt diff --git a/tests/etl_test/data2_test1.py b/tests/e2e/etl/data2_test1.py similarity index 100% rename from tests/etl_test/data2_test1.py rename to tests/e2e/etl/data2_test1.py diff --git a/tests/etl_test/data2_test1_result.txt b/tests/e2e/etl/data2_test1_result.txt similarity index 100% rename from tests/etl_test/data2_test1_result.txt rename to tests/e2e/etl/data2_test1_result.txt diff --git a/tests/etl_test/data2_test1_result_bk.txt b/tests/e2e/etl/data2_test1_result_bk.txt similarity index 100% rename from tests/etl_test/data2_test1_result_bk.txt rename to tests/e2e/etl/data2_test1_result_bk.txt diff --git a/tests/etl_test/data2_test2.py b/tests/e2e/etl/data2_test2.py similarity index 100% rename from tests/etl_test/data2_test2.py rename to tests/e2e/etl/data2_test2.py diff --git a/tests/etl_test/data2_test3.py b/tests/e2e/etl/data2_test3.py similarity index 100% rename from tests/etl_test/data2_test3.py rename to tests/e2e/etl/data2_test3.py diff --git a/tests/etl_test/data3.txt b/tests/e2e/etl/data3.txt similarity index 100% rename from tests/etl_test/data3.txt rename to tests/e2e/etl/data3.txt diff --git a/tests/etl_test/data3_test1.py b/tests/e2e/etl/data3_test1.py similarity index 100% rename from tests/etl_test/data3_test1.py rename to tests/e2e/etl/data3_test1.py diff --git a/tests/etl_test/data3_test1_result.txt b/tests/e2e/etl/data3_test1_result.txt similarity index 100% rename from tests/etl_test/data3_test1_result.txt rename to tests/e2e/etl/data3_test1_result.txt diff --git a/tests/etl_test/data4.txt b/tests/e2e/etl/data4.txt similarity index 100% rename from tests/etl_test/data4.txt rename to tests/e2e/etl/data4.txt diff --git a/tests/etl_test/data4_lookup_csv1.txt b/tests/e2e/etl/data4_lookup_csv1.txt similarity index 100% rename from tests/etl_test/data4_lookup_csv1.txt rename to tests/e2e/etl/data4_lookup_csv1.txt diff --git a/tests/etl_test/data4_test1.py b/tests/e2e/etl/data4_test1.py similarity index 100% rename from tests/etl_test/data4_test1.py rename to tests/e2e/etl/data4_test1.py diff --git a/tests/etl_test/data4_test1_result.txt b/tests/e2e/etl/data4_test1_result.txt similarity index 100% rename from tests/etl_test/data4_test1_result.txt rename to tests/e2e/etl/data4_test1_result.txt diff --git a/tests/etl_test/etl_test.py b/tests/e2e/etl/etl_test.py similarity index 98% rename from tests/etl_test/etl_test.py rename to tests/e2e/etl/etl_test.py index 063bbe72..f2516b03 100644 --- a/tests/etl_test/etl_test.py +++ b/tests/e2e/etl/etl_test.py @@ -1,3 +1,6 @@ +import pytest +pytestmark = pytest.mark.e2e + import time from aliyun.log import * diff --git a/tests/etl_test/json_data/CVE-2013-0169.json b/tests/e2e/etl/json_data/CVE-2013-0169.json similarity index 100% rename from tests/etl_test/json_data/CVE-2013-0169.json rename to tests/e2e/etl/json_data/CVE-2013-0169.json diff --git a/tests/etl_test/json_data/simple_data.json b/tests/e2e/etl/json_data/simple_data.json similarity index 100% rename from tests/etl_test/json_data/simple_data.json rename to tests/e2e/etl/json_data/simple_data.json diff --git a/tests/etl_test/sls_config_1.py b/tests/e2e/etl/sls_config_1.py similarity index 100% rename from tests/etl_test/sls_config_1.py rename to tests/e2e/etl/sls_config_1.py diff --git a/tests/etl_test/test_case.py b/tests/e2e/etl/test_case.py similarity index 99% rename from tests/etl_test/test_case.py rename to tests/e2e/etl/test_case.py index 1e59eebc..48a9828b 100644 --- a/tests/etl_test/test_case.py +++ b/tests/e2e/etl/test_case.py @@ -1,5 +1,8 @@ #encoding: utf8 +import pytest +pytestmark = pytest.mark.e2e + from aliyun.log.etl_core import * from aliyun.log.etl_core.config_parser import ConfigParser from aliyun.log.etl_core.runner import Runner @@ -1186,24 +1189,25 @@ def test_zip(): assert t( {"combine": ZIP("f1", "f2", lparse=(",", '|'), rparse=(",", '#'))})({"f1": '|a,a|, b, |c,c|', "f2":'x, #y,y#, z'}) == {"f1": '|a,a|, b, |c,c|', "f2":'x, #y,y#, z', 'combine': '"a,a#x","b#y,y","c,c#z"'} -test_condition() -test_condition_not() -test_v() -test_regex() -test_csv() -test_lookup_dict() -test_lookup_load_csv() -test_lookup_mapping() -test_kv() -test_zip() -test_split() -test_split_filter() -test_json_expand() -test_json_match() -test_json_filter() -test_json_mixed() -test_dispatch_transform() -test_meta() -test_parse() -test_runner() -test_module() +if __name__ == "__main__": + test_condition() + test_condition_not() + test_v() + test_regex() + test_csv() + test_lookup_dict() + test_lookup_load_csv() + test_lookup_mapping() + test_kv() + test_zip() + test_split() + test_split_filter() + test_json_expand() + test_json_match() + test_json_filter() + test_json_mixed() + test_dispatch_transform() + test_meta() + test_parse() + test_runner() + test_module() diff --git a/tests/e2e/ingestion/__init__.py b/tests/e2e/ingestion/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/ingestion_test/elasticsearch_test.py b/tests/e2e/ingestion/elasticsearch_test.py similarity index 98% rename from tests/ingestion_test/elasticsearch_test.py rename to tests/e2e/ingestion/elasticsearch_test.py index 2f025f2b..ba837585 100644 --- a/tests/ingestion_test/elasticsearch_test.py +++ b/tests/e2e/ingestion/elasticsearch_test.py @@ -1,4 +1,7 @@ # encoding: utf-8 +import pytest +pytestmark = pytest.mark.e2e + import json import os import time diff --git a/tests/ingestion_test/kafka_test.py b/tests/e2e/ingestion/kafka_test.py similarity index 98% rename from tests/ingestion_test/kafka_test.py rename to tests/e2e/ingestion/kafka_test.py index a753152d..18a0b2f9 100644 --- a/tests/ingestion_test/kafka_test.py +++ b/tests/e2e/ingestion/kafka_test.py @@ -1,4 +1,7 @@ # encoding: utf-8 +import pytest +pytestmark = pytest.mark.e2e + import json import os import time diff --git a/tests/ingestion_test/oss_test.py b/tests/e2e/ingestion/oss_test.py similarity index 98% rename from tests/ingestion_test/oss_test.py rename to tests/e2e/ingestion/oss_test.py index f5535ede..4985af76 100644 --- a/tests/ingestion_test/oss_test.py +++ b/tests/e2e/ingestion/oss_test.py @@ -1,4 +1,7 @@ # encoding: utf-8 +import pytest +pytestmark = pytest.mark.e2e + import json import os import time diff --git a/tests/e2e/logclient/__init__.py b/tests/e2e/logclient/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/logclient/test_get_logs.py b/tests/e2e/logclient/test_get_logs.py similarity index 99% rename from tests/logclient/test_get_logs.py rename to tests/e2e/logclient/test_get_logs.py index 79c7abcc..a92a12d2 100644 --- a/tests/logclient/test_get_logs.py +++ b/tests/e2e/logclient/test_get_logs.py @@ -1,3 +1,6 @@ +import pytest +pytestmark = pytest.mark.e2e + import unittest import os diff --git a/tests/logclient/test_project.py b/tests/e2e/logclient/test_project.py similarity index 93% rename from tests/logclient/test_project.py rename to tests/e2e/logclient/test_project.py index b3b19a0f..aaa2b0f2 100644 --- a/tests/logclient/test_project.py +++ b/tests/e2e/logclient/test_project.py @@ -1,3 +1,6 @@ +import pytest +pytestmark = pytest.mark.e2e + from aliyun.log import LogClient diff --git a/tests/logclient/test_resource_group.py b/tests/e2e/logclient/test_resource_group.py similarity index 96% rename from tests/logclient/test_resource_group.py rename to tests/e2e/logclient/test_resource_group.py index 1cb00edf..666299f0 100644 --- a/tests/logclient/test_resource_group.py +++ b/tests/e2e/logclient/test_resource_group.py @@ -1,3 +1,6 @@ +import pytest +pytestmark = pytest.mark.e2e + import os from aliyun.log import LogClient diff --git a/tests/logclient/test_tags.py b/tests/e2e/logclient/test_tags.py similarity index 97% rename from tests/logclient/test_tags.py rename to tests/e2e/logclient/test_tags.py index 7c0c5531..56535cac 100644 --- a/tests/logclient/test_tags.py +++ b/tests/e2e/logclient/test_tags.py @@ -1,3 +1,6 @@ +import pytest +pytestmark = pytest.mark.e2e + import os from aliyun.log import LogClient diff --git a/tests/integration_test/logging.conf b/tests/e2e/logging.conf similarity index 100% rename from tests/integration_test/logging.conf rename to tests/e2e/logging.conf diff --git a/tests/e2e/resource/__init__.py b/tests/e2e/resource/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/resource_test/resource_test.py b/tests/e2e/resource/resource_test.py similarity index 99% rename from tests/resource_test/resource_test.py rename to tests/e2e/resource/resource_test.py index bf8cb133..584c662d 100644 --- a/tests/resource_test/resource_test.py +++ b/tests/e2e/resource/resource_test.py @@ -1,3 +1,6 @@ +import pytest +pytestmark = pytest.mark.e2e + from aliyun.log import LogClient, LogException from aliyun.log.resource_params import * import unittest diff --git a/tests/e2e/schedule_sql/__init__.py b/tests/e2e/schedule_sql/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/schedule_sql_test/schedule_sql_demo_test.py b/tests/e2e/schedule_sql/schedule_sql_demo_test.py similarity index 99% rename from tests/schedule_sql_test/schedule_sql_demo_test.py rename to tests/e2e/schedule_sql/schedule_sql_demo_test.py index 6014b526..74579dcb 100644 --- a/tests/schedule_sql_test/schedule_sql_demo_test.py +++ b/tests/e2e/schedule_sql/schedule_sql_demo_test.py @@ -1,3 +1,6 @@ +import pytest +pytestmark = pytest.mark.e2e + import time from aliyun.log import LogClient diff --git a/tests/test_alert.py b/tests/e2e/test_alert.py similarity index 95% rename from tests/test_alert.py rename to tests/e2e/test_alert.py index cd23f6c4..7f1b57d3 100644 --- a/tests/test_alert.py +++ b/tests/e2e/test_alert.py @@ -1,6 +1,9 @@ # encoding: utf-8 from __future__ import print_function +import pytest +pytestmark = pytest.mark.e2e + from aliyun.log import * import os diff --git a/tests/compress_test.py b/tests/e2e/test_compress_e2e.py similarity index 70% rename from tests/compress_test.py rename to tests/e2e/test_compress_e2e.py index fd77f881..7a161a01 100644 --- a/tests/compress_test.py +++ b/tests/e2e/test_compress_e2e.py @@ -1,28 +1,27 @@ +"""End-to-end tests for compress paths against a real SLS endpoint. + +The pure ``test_lz4`` round-trip lives in :mod:`tests.unit.test_compress`. +""" import time -from aliyun.log.compress import Compressor, CompressType -from aliyun.log import LogClient, PutLogsRequest, LogItem, LogGroup +import pytest + +from aliyun.log import LogClient, PutLogsRequest, LogItem, LogGroup -def test_lz4(): - text = b'sadsadsa189634o2??ASBKHD' - compressed = Compressor.compress(text, CompressType.LZ4) - raw_size = len(text) - uncompressed = Compressor.decompress( - compressed, raw_size, CompressType.LZ4) +from tests._helpers.env import require_sls_env - assert text == uncompressed, "The decompressed data does not match the original" +pytestmark = pytest.mark.e2e -# setup -project = '' -logstore = '' -access_key_id = '' -access_key_secret = '' -endpoint = '' -client = LogClient(endpoint, access_key_id, access_key_secret) +@pytest.fixture +def client_env(): + env = require_sls_env() + client = LogClient(env["endpoint"], env["access_key_id"], env["access_key_secret"]) + return client, env["project"], env["logstore"] -def test_pull_logs(): +def test_pull_logs(client_env): + client, project, logstore = client_env shards = client.list_shards(project, logstore).get_shards_info() shard = shards[0]['shardID'] cursor = client.get_cursor(project, logstore, shard, 'begin').get_cursor() @@ -32,7 +31,8 @@ def test_pull_logs(): client.pull_logs(project, logstore, shard, cursor, compress=False) -def test_put_log_raw(): +def test_put_log_raw(client_env): + client, project, logstore = client_env log_group = LogGroup() log = log_group.Logs.add() log.Time = int(time.time()) @@ -45,7 +45,8 @@ def test_put_log_raw(): client.put_log_raw(project, logstore, log_group, None) -def test_put_logs(): +def test_put_logs(client_env): + client, project, logstore = client_env client.put_logs(PutLogsRequest(project, logstore, 'test', '', [LogItem(contents=[('ghello', 'test')])], compress=True)) client.put_logs(PutLogsRequest(project, logstore, 'test', @@ -54,7 +55,8 @@ def test_put_logs(): '', [LogItem(contents=[('ghello', 'test')])], compress=False)) -def test_get_log(): +def test_get_log(client_env): + client, project, logstore = client_env client._get_logs_v2_enabled = True from_time = int(time.time()) - 100 to_time = int(time.time()) diff --git a/tests/integration_test/test_entity.py b/tests/e2e/test_entity.py similarity index 96% rename from tests/integration_test/test_entity.py rename to tests/e2e/test_entity.py index 3e5e6a60..3b44944e 100755 --- a/tests/integration_test/test_entity.py +++ b/tests/e2e/test_entity.py @@ -1,199 +1,202 @@ -# encoding: utf-8 -from __future__ import print_function - -from aliyun.log import * -import time -import os - -dashboard_detail = { - "charts": [ - { - "display": { - "displayName": "", - "height": 5, - "width": 5, - "xAxis": [ - "province" - ], - "xPos": 0, - "yAxis": [ - "pv" - ], - "yPos": 0 - }, - "search": { - "end": "now", - "logstore": "access-log", - "query": "method: GET | select ip_to_province(remote_addr) as province , count(1) as pv group by province order by pv desc ", - "start": "-86400s", - "topic": "" - }, - "title": "map", - "type": "map" - }, - { - "display": { - "displayName": "", - "height": 5, - "width": 5, - "xAxis": [ - "province" - ], - "xPos": 5, - "yAxis": [ - "pv" - ], - "yPos": 0 - }, - "search": { - "end": "now", - "logstore": "access-log", - "query": "method: POST | select ip_to_province(remote_addr) as province , count(1) as pv group by province order by pv desc ", - "start": "-86400s", - "topic": "" - }, - "title": "post_map", - "type": "map" - } - ], - "dashboardName": 'alert_' + str(time.time()).replace('.', '-'), - "description": "" -} - -dashboard_for_alert = 'dashboard_' + str(time.time()).replace('.', '-') - -alert_detail = { - "name": 'alert_' + str(time.time()).replace('.', '-'), - "displayName": "Alert for testing", - "description": "", - "type": "Alert", - "state": "Enabled", - "schedule": { - "type": "FixedRate", - "interval": "5m", - }, - "configuration": { - "condition": "total >= 100", - "dashboard": dashboard_for_alert, - "queryList": [ - { - "logStore": "test-logstore", - "start": "-120s", - "end": "now", - "timeSpanType": "Custom", - "chartTitle": "chart-test", - "query": "* | select count(1) as total", - } - ], - "notificationList": [ - { - "type": "DingTalk", - "atMobiles": ['1867393xxxx'], - "serviceUri": "https://oapi.dingtalk.com/robot/send?access_token=xxxx", - "content": "Hi @1867393xxxx, your alert ${AlertDisplayName} is fired", - }, - { - "type": "MessageCenter", - "content": "Message", - }, - { - "type": "Email", - "subject": "Alerting", - "emailList": ["abc@test.com"], - "content": "Email Message", - }, - { - "type": "SMS", - "mobileList": ["132373830xx"], - "content": "Cellphone message" - }, - { - "type": "Voice", - "mobileList": ["132373830xx"], - "content": "Voice notification...", - } - ], - "muteUntil": int(time.time()) + 300, - "notifyThreshold": 1, - "throttling": "5m", - } -} - -savedsearch_detail = { - "logstore": "test2", - "savedsearchName": 'search_' + str(time.time()).replace('.', '-'), - "searchQuery": "boy | select sex, count() as Count group by sex", - "topic": "" -} - - -def main(): - endpoint = os.environ.get('ALIYUN_LOG_SAMPLE_ENDPOINT', '') - accessKeyId = os.environ.get('ALIYUN_LOG_SAMPLE_ACCESSID', '') - accessKey = os.environ.get('ALIYUN_LOG_SAMPLE_ACCESSKEY', '') - project = os.environ.get('ALIYUN_LOG_SAMPLE_PROJECT', '') - - dashboard = dashboard_detail.get('dashboardName') - - client = LogClient(endpoint, accessKeyId, accessKey, "") - - res = client.create_dashboard(project, dashboard_detail) - res.log_print() - - res = client.list_dashboard(project) - res.log_print() - - res = client.get_dashboard(project, dashboard) - res.log_print() - - res = client.update_dashboard(project, dashboard_detail) - res.log_print() - - res = client.delete_dashboard(project, dashboard) - res.log_print() - - dashboard_detail['dashboardName'] = dashboard_for_alert - res = client.create_dashboard(project, dashboard_detail) - res.log_print() - - alert = alert_detail.get('name') - - res = client.create_alert(project, alert_detail) - res.log_print() - - res = client.list_alert(project) - res.log_print() - print(res.get_alerts()) - - res = client.get_alert(project, alert) - res.log_print() - - res = client.update_alert(project, alert_detail) - res.log_print() - - res = client.delete_alert(project, alert) - res.log_print() - res = client.delete_dashboard(project, dashboard_for_alert) - res.log_print() - - savedsearch = savedsearch_detail.get('savedsearchName') - - res = client.create_savedsearch(project, savedsearch_detail) - res.log_print() - - res = client.list_savedsearch(project) - res.log_print() - print(res.get_savedsearches()) - - res = client.get_savedsearch(project, savedsearch) - res.log_print() - - res = client.update_savedsearch(project, savedsearch_detail) - res.log_print() - - res = client.delete_savedsearch(project, savedsearch) - res.log_print() - - -if __name__ == '__main__': - main() +# encoding: utf-8 +from __future__ import print_function + +import pytest +pytestmark = pytest.mark.e2e + +from aliyun.log import * +import time +import os + +dashboard_detail = { + "charts": [ + { + "display": { + "displayName": "", + "height": 5, + "width": 5, + "xAxis": [ + "province" + ], + "xPos": 0, + "yAxis": [ + "pv" + ], + "yPos": 0 + }, + "search": { + "end": "now", + "logstore": "access-log", + "query": "method: GET | select ip_to_province(remote_addr) as province , count(1) as pv group by province order by pv desc ", + "start": "-86400s", + "topic": "" + }, + "title": "map", + "type": "map" + }, + { + "display": { + "displayName": "", + "height": 5, + "width": 5, + "xAxis": [ + "province" + ], + "xPos": 5, + "yAxis": [ + "pv" + ], + "yPos": 0 + }, + "search": { + "end": "now", + "logstore": "access-log", + "query": "method: POST | select ip_to_province(remote_addr) as province , count(1) as pv group by province order by pv desc ", + "start": "-86400s", + "topic": "" + }, + "title": "post_map", + "type": "map" + } + ], + "dashboardName": 'alert_' + str(time.time()).replace('.', '-'), + "description": "" +} + +dashboard_for_alert = 'dashboard_' + str(time.time()).replace('.', '-') + +alert_detail = { + "name": 'alert_' + str(time.time()).replace('.', '-'), + "displayName": "Alert for testing", + "description": "", + "type": "Alert", + "state": "Enabled", + "schedule": { + "type": "FixedRate", + "interval": "5m", + }, + "configuration": { + "condition": "total >= 100", + "dashboard": dashboard_for_alert, + "queryList": [ + { + "logStore": "test-logstore", + "start": "-120s", + "end": "now", + "timeSpanType": "Custom", + "chartTitle": "chart-test", + "query": "* | select count(1) as total", + } + ], + "notificationList": [ + { + "type": "DingTalk", + "atMobiles": ['1867393xxxx'], + "serviceUri": "https://oapi.dingtalk.com/robot/send?access_token=xxxx", + "content": "Hi @1867393xxxx, your alert ${AlertDisplayName} is fired", + }, + { + "type": "MessageCenter", + "content": "Message", + }, + { + "type": "Email", + "subject": "Alerting", + "emailList": ["abc@test.com"], + "content": "Email Message", + }, + { + "type": "SMS", + "mobileList": ["132373830xx"], + "content": "Cellphone message" + }, + { + "type": "Voice", + "mobileList": ["132373830xx"], + "content": "Voice notification...", + } + ], + "muteUntil": int(time.time()) + 300, + "notifyThreshold": 1, + "throttling": "5m", + } +} + +savedsearch_detail = { + "logstore": "test2", + "savedsearchName": 'search_' + str(time.time()).replace('.', '-'), + "searchQuery": "boy | select sex, count() as Count group by sex", + "topic": "" +} + + +def main(): + endpoint = os.environ.get('ALIYUN_LOG_SAMPLE_ENDPOINT', '') + accessKeyId = os.environ.get('ALIYUN_LOG_SAMPLE_ACCESSID', '') + accessKey = os.environ.get('ALIYUN_LOG_SAMPLE_ACCESSKEY', '') + project = os.environ.get('ALIYUN_LOG_SAMPLE_PROJECT', '') + + dashboard = dashboard_detail.get('dashboardName') + + client = LogClient(endpoint, accessKeyId, accessKey, "") + + res = client.create_dashboard(project, dashboard_detail) + res.log_print() + + res = client.list_dashboard(project) + res.log_print() + + res = client.get_dashboard(project, dashboard) + res.log_print() + + res = client.update_dashboard(project, dashboard_detail) + res.log_print() + + res = client.delete_dashboard(project, dashboard) + res.log_print() + + dashboard_detail['dashboardName'] = dashboard_for_alert + res = client.create_dashboard(project, dashboard_detail) + res.log_print() + + alert = alert_detail.get('name') + + res = client.create_alert(project, alert_detail) + res.log_print() + + res = client.list_alert(project) + res.log_print() + print(res.get_alerts()) + + res = client.get_alert(project, alert) + res.log_print() + + res = client.update_alert(project, alert_detail) + res.log_print() + + res = client.delete_alert(project, alert) + res.log_print() + res = client.delete_dashboard(project, dashboard_for_alert) + res.log_print() + + savedsearch = savedsearch_detail.get('savedsearchName') + + res = client.create_savedsearch(project, savedsearch_detail) + res.log_print() + + res = client.list_savedsearch(project) + res.log_print() + print(res.get_savedsearches()) + + res = client.get_savedsearch(project, savedsearch) + res.log_print() + + res = client.update_savedsearch(project, savedsearch_detail) + res.log_print() + + res = client.delete_savedsearch(project, savedsearch) + res.log_print() + + +if __name__ == '__main__': + main() diff --git a/tests/integration_test/test_log_handler.py b/tests/e2e/test_log_handler.py similarity index 98% rename from tests/integration_test/test_log_handler.py rename to tests/e2e/test_log_handler.py index ef01366f..92a5155a 100644 --- a/tests/integration_test/test_log_handler.py +++ b/tests/e2e/test_log_handler.py @@ -1,7 +1,10 @@ #encoding: utf8 +import pytest +pytestmark = pytest.mark.e2e + from aliyun.log.logger_hanlder import LogFields, QueuedLogHandler -from test_logtail_config import clean_project +from .test_logtail_config import clean_project from aliyun.log import LogClient import logging import logging.config diff --git a/tests/integration_test/test_logtail_config.py b/tests/e2e/test_logtail_config.py similarity index 97% rename from tests/integration_test/test_logtail_config.py rename to tests/e2e/test_logtail_config.py index 831e7733..09b006df 100755 --- a/tests/integration_test/test_logtail_config.py +++ b/tests/e2e/test_logtail_config.py @@ -1,104 +1,107 @@ -# encoding: utf-8 -from __future__ import print_function - -from aliyun.log import * -import time -import os -import json -from aliyun.log.logtail_config_detail import LogtailConfigGenerator -import os - - -def clean_project(client, project): - - print("#" * 1000) - print("*** start to cleanup project: ", project) - - print("*** start to delete logtail config") - stores = client.list_logstores(ListLogstoresRequest(project)) - n = 0 - logtails = client.list_logtail_config(project, size=-1) - for logtail in logtails.get_configs(): - try: - ret = client.delete_logtail_config(project, logtail) - print(ret) - n += 1 - except LogException as ex: - print("skip deleting config for", ex) - print("### deleted {0} logtail config".format(n)) - - print("*** start to delete logstore") - for l in stores.get_logstores(): - client.delete_logstore(project, l) - print("### deleted {0} log store".format(len(stores.get_logstores()))) - - # delete project - client.delete_project(project) - - -def test_logtail_config(client, project): - dir_path = os.sep.join([os.path.dirname(__file__), "data"]) - file_names = [ - 'simple_1', 'simple_2', 'simple_3', 'simple_4_docker', - 'feitian_1', 'feitian_2', - 'json_1', 'json_2', 'json_3', 'json_4_docker', - 'ngnix_1', - 'reg_1', 'reg_2', 'reg_3', 'reg_4_docker', - 'sep_1', 'sep_2', 'sep_3','sep_4_docker', - 'syslog_1', - 'docker-stdout-config', 'mysql-binlog-config', - 'mysql-rawsql-config', 'nginx-status-config' - ] - - for file_name in file_names: - json_path = os.sep.join([dir_path, file_name + '.json']) - with open(json_path, "r") as f: - json_value = json.load(f) - detail = LogtailConfigGenerator.generate_config(json_value) - print("****create config", file_name) - res = client.create_logtail_config(project, detail) - res.log_print() - - time.sleep(20) - for config_name in file_names: - print("****get config", config_name) - res = client.get_logtail_config(project, config_name) - res.log_print() - - for file_name in file_names: - json_path = os.sep.join([dir_path, file_name + '.json']) - with open(json_path, "r") as f: - json_value = json.load(f) - detail = LogtailConfigGenerator.generate_config(json_value) - print("****update config", file_name) - res = client.update_logtail_config(project, detail) - res.log_print() - -def main(): - endpoint = os.environ.get('ALIYUN_LOG_SAMPLE_ENDPOINT', '') - accessKeyId = os.environ.get('ALIYUN_LOG_SAMPLE_ACCESSID', '') - accessKey = os.environ.get('ALIYUN_LOG_SAMPLE_ACCESSKEY', '') - - project = 'python-sdk-test' + str(time.time()).replace('.', '-') - logstore = 'logstore' - - assert endpoint and accessKeyId and accessKey, ValueError("endpoint/access_id/key cannot be empty") - - client = LogClient(endpoint, accessKeyId, accessKey, "") - - print("****create project", project) - client.create_project(project, "SDK test") - time.sleep(10) - - try: - print("****create logstore", logstore) - client.create_logstore(project, logstore, 1, 1) - time.sleep(40) - - test_logtail_config(client, project) - finally: - clean_project(client, project) - - -if __name__ == '__main__': - main() +# encoding: utf-8 +from __future__ import print_function + +import pytest +pytestmark = pytest.mark.e2e + +from aliyun.log import * +import time +import os +import json +from aliyun.log.logtail_config_detail import LogtailConfigGenerator +import os + + +def clean_project(client, project): + + print("#" * 1000) + print("*** start to cleanup project: ", project) + + print("*** start to delete logtail config") + stores = client.list_logstores(ListLogstoresRequest(project)) + n = 0 + logtails = client.list_logtail_config(project, size=-1) + for logtail in logtails.get_configs(): + try: + ret = client.delete_logtail_config(project, logtail) + print(ret) + n += 1 + except LogException as ex: + print("skip deleting config for", ex) + print("### deleted {0} logtail config".format(n)) + + print("*** start to delete logstore") + for l in stores.get_logstores(): + client.delete_logstore(project, l) + print("### deleted {0} log store".format(len(stores.get_logstores()))) + + # delete project + client.delete_project(project) + + +def test_logtail_config(client, project): + dir_path = os.sep.join([os.path.dirname(__file__), "data"]) + file_names = [ + 'simple_1', 'simple_2', 'simple_3', 'simple_4_docker', + 'feitian_1', 'feitian_2', + 'json_1', 'json_2', 'json_3', 'json_4_docker', + 'ngnix_1', + 'reg_1', 'reg_2', 'reg_3', 'reg_4_docker', + 'sep_1', 'sep_2', 'sep_3','sep_4_docker', + 'syslog_1', + 'docker-stdout-config', 'mysql-binlog-config', + 'mysql-rawsql-config', 'nginx-status-config' + ] + + for file_name in file_names: + json_path = os.sep.join([dir_path, file_name + '.json']) + with open(json_path, "r") as f: + json_value = json.load(f) + detail = LogtailConfigGenerator.generate_config(json_value) + print("****create config", file_name) + res = client.create_logtail_config(project, detail) + res.log_print() + + time.sleep(20) + for config_name in file_names: + print("****get config", config_name) + res = client.get_logtail_config(project, config_name) + res.log_print() + + for file_name in file_names: + json_path = os.sep.join([dir_path, file_name + '.json']) + with open(json_path, "r") as f: + json_value = json.load(f) + detail = LogtailConfigGenerator.generate_config(json_value) + print("****update config", file_name) + res = client.update_logtail_config(project, detail) + res.log_print() + +def main(): + endpoint = os.environ.get('ALIYUN_LOG_SAMPLE_ENDPOINT', '') + accessKeyId = os.environ.get('ALIYUN_LOG_SAMPLE_ACCESSID', '') + accessKey = os.environ.get('ALIYUN_LOG_SAMPLE_ACCESSKEY', '') + + project = 'python-sdk-test' + str(time.time()).replace('.', '-') + logstore = 'logstore' + + assert endpoint and accessKeyId and accessKey, ValueError("endpoint/access_id/key cannot be empty") + + client = LogClient(endpoint, accessKeyId, accessKey, "") + + print("****create project", project) + client.create_project(project, "SDK test") + time.sleep(10) + + try: + print("****create logstore", logstore) + client.create_logstore(project, logstore, 1, 1) + time.sleep(40) + + test_logtail_config(client, project) + finally: + clean_project(client, project) + + +if __name__ == '__main__': + main() diff --git a/tests/examples/README.md b/tests/examples/README.md new file mode 100644 index 00000000..254e71d8 --- /dev/null +++ b/tests/examples/README.md @@ -0,0 +1,15 @@ +# tests/examples/ + +Topic-grouped example scripts demonstrating common SLS workflows. These are +**not** collected by `pytest` and are not run in CI. + +Folders: + +- `consumer_group_examples/` — consumer group patterns (keyword monitor, + copy data, sync to splunk/syslog). +- `export_examples/` — sinks for ODPS / OSS. +- `rebuild_index_examples/` — re-index helpers. +- `jupyter_magic_test/` — Jupyter notebooks for the `%log` magic. +- `test_migration_manager*.py` — long-running migration scripts. + +Each script reads its own credentials/config; treat them as walkthroughs. diff --git a/tests/consumer_group_examples/client_worker_with_query.py b/tests/examples/consumer_group_examples/client_worker_with_query.py similarity index 100% rename from tests/consumer_group_examples/client_worker_with_query.py rename to tests/examples/consumer_group_examples/client_worker_with_query.py diff --git a/tests/consumer_group_examples/copy_data_to_logstore.py b/tests/examples/consumer_group_examples/copy_data_to_logstore.py similarity index 100% rename from tests/consumer_group_examples/copy_data_to_logstore.py rename to tests/examples/consumer_group_examples/copy_data_to_logstore.py diff --git a/tests/consumer_group_examples/keyword_monitor.py b/tests/examples/consumer_group_examples/keyword_monitor.py similarity index 100% rename from tests/consumer_group_examples/keyword_monitor.py rename to tests/examples/consumer_group_examples/keyword_monitor.py diff --git a/tests/consumer_group_examples/keyword_monitor_multiple_logstores.py b/tests/examples/consumer_group_examples/keyword_monitor_multiple_logstores.py similarity index 100% rename from tests/consumer_group_examples/keyword_monitor_multiple_logstores.py rename to tests/examples/consumer_group_examples/keyword_monitor_multiple_logstores.py diff --git a/tests/consumer_group_examples/sync_data_to_splunk.py b/tests/examples/consumer_group_examples/sync_data_to_splunk.py similarity index 100% rename from tests/consumer_group_examples/sync_data_to_splunk.py rename to tests/examples/consumer_group_examples/sync_data_to_splunk.py diff --git a/tests/consumer_group_examples/sync_data_to_splunk_multiple_logstores.py b/tests/examples/consumer_group_examples/sync_data_to_splunk_multiple_logstores.py similarity index 100% rename from tests/consumer_group_examples/sync_data_to_splunk_multiple_logstores.py rename to tests/examples/consumer_group_examples/sync_data_to_splunk_multiple_logstores.py diff --git a/tests/consumer_group_examples/sync_data_to_syslog.py b/tests/examples/consumer_group_examples/sync_data_to_syslog.py similarity index 100% rename from tests/consumer_group_examples/sync_data_to_syslog.py rename to tests/examples/consumer_group_examples/sync_data_to_syslog.py diff --git a/tests/consumer_group_examples/syslogclient.py b/tests/examples/consumer_group_examples/syslogclient.py similarity index 100% rename from tests/consumer_group_examples/syslogclient.py rename to tests/examples/consumer_group_examples/syslogclient.py diff --git a/tests/export_examples/export_odps_sink_demo.py b/tests/examples/export_examples/export_odps_sink_demo.py similarity index 100% rename from tests/export_examples/export_odps_sink_demo.py rename to tests/examples/export_examples/export_odps_sink_demo.py diff --git a/tests/export_examples/export_oss_sink_demo.py b/tests/examples/export_examples/export_oss_sink_demo.py similarity index 100% rename from tests/export_examples/export_oss_sink_demo.py rename to tests/examples/export_examples/export_oss_sink_demo.py diff --git a/tests/jupyter_magic_test/magic_ext_usage_demo.ipynb b/tests/examples/jupyter_magic_test/magic_ext_usage_demo.ipynb similarity index 100% rename from tests/jupyter_magic_test/magic_ext_usage_demo.ipynb rename to tests/examples/jupyter_magic_test/magic_ext_usage_demo.ipynb diff --git a/tests/jupyter_magic_test/save_to_excel_demo.ipynb b/tests/examples/jupyter_magic_test/save_to_excel_demo.ipynb similarity index 100% rename from tests/jupyter_magic_test/save_to_excel_demo.ipynb rename to tests/examples/jupyter_magic_test/save_to_excel_demo.ipynb diff --git a/tests/rebuild_index_examples/rebuild_index_example.py b/tests/examples/rebuild_index_examples/rebuild_index_example.py similarity index 100% rename from tests/rebuild_index_examples/rebuild_index_example.py rename to tests/examples/rebuild_index_examples/rebuild_index_example.py diff --git a/tests/es_migration/test_migration_manager.py b/tests/examples/test_migration_manager.py similarity index 100% rename from tests/es_migration/test_migration_manager.py rename to tests/examples/test_migration_manager.py diff --git a/tests/es_migration/test_migration_manager_with_logclient.py b/tests/examples/test_migration_manager_with_logclient.py similarity index 100% rename from tests/es_migration/test_migration_manager_with_logclient.py rename to tests/examples/test_migration_manager_with_logclient.py diff --git a/tests/samples/README.md b/tests/samples/README.md new file mode 100644 index 00000000..26dec57e --- /dev/null +++ b/tests/samples/README.md @@ -0,0 +1,15 @@ +# tests/samples/ + +Self-contained scripts that show how to call SLS APIs. They are **not** +collected by `pytest` (see `pyproject.toml`'s `testpaths`) and they are not +run in CI. + +Most read credentials from the legacy `ALIYUN_LOG_SAMPLE_*` env vars and call +into a real endpoint. Treat them as runnable examples, not as tests. + +``` +python tests/samples/sample.py +``` + +If you are looking for a starter snippet, prefer `tests/examples/` for the +folder of curated, topic-grouped examples. diff --git a/tests/sample.py b/tests/samples/sample.py similarity index 100% rename from tests/sample.py rename to tests/samples/sample.py diff --git a/tests/sample2.py b/tests/samples/sample2.py similarity index 100% rename from tests/sample2.py rename to tests/samples/sample2.py diff --git a/tests/sample3.py b/tests/samples/sample3.py similarity index 100% rename from tests/sample3.py rename to tests/samples/sample3.py diff --git a/tests/sample_consumer.py b/tests/samples/sample_consumer.py similarity index 100% rename from tests/sample_consumer.py rename to tests/samples/sample_consumer.py diff --git a/tests/sample_consumer2.py b/tests/samples/sample_consumer2.py similarity index 100% rename from tests/sample_consumer2.py rename to tests/samples/sample_consumer2.py diff --git a/tests/sample_data_redundancy.py b/tests/samples/sample_data_redundancy.py similarity index 100% rename from tests/sample_data_redundancy.py rename to tests/samples/sample_data_redundancy.py diff --git a/tests/sample_getcontextlogs.py b/tests/samples/sample_getcontextlogs.py similarity index 100% rename from tests/sample_getcontextlogs.py rename to tests/samples/sample_getcontextlogs.py diff --git a/tests/sample_ingestion.py b/tests/samples/sample_ingestion.py similarity index 100% rename from tests/sample_ingestion.py rename to tests/samples/sample_ingestion.py diff --git a/tests/sample_metric_agg_rules.py b/tests/samples/sample_metric_agg_rules.py similarity index 100% rename from tests/sample_metric_agg_rules.py rename to tests/samples/sample_metric_agg_rules.py diff --git a/tests/sample_metric_store.py b/tests/samples/sample_metric_store.py similarity index 100% rename from tests/sample_metric_store.py rename to tests/samples/sample_metric_store.py diff --git a/tests/sample_object_api.py b/tests/samples/sample_object_api.py similarity index 100% rename from tests/sample_object_api.py rename to tests/samples/sample_object_api.py diff --git a/tests/sample_pipeline_config.py b/tests/samples/sample_pipeline_config.py similarity index 100% rename from tests/sample_pipeline_config.py rename to tests/samples/sample_pipeline_config.py diff --git a/tests/sample_shipper.py b/tests/samples/sample_shipper.py similarity index 100% rename from tests/sample_shipper.py rename to tests/samples/sample_shipper.py diff --git a/tests/sample_sql.py b/tests/samples/sample_sql.py similarity index 100% rename from tests/sample_sql.py rename to tests/samples/sample_sql.py diff --git a/tests/sample_v4_sign.py b/tests/samples/sample_v4_sign.py similarity index 100% rename from tests/sample_v4_sign.py rename to tests/samples/sample_v4_sign.py diff --git a/tests/user.csv b/tests/samples/user.csv similarity index 100% rename from tests/user.csv rename to tests/samples/user.csv diff --git a/tests/cli_config_check/restrict_config_test_data/config1_fail.py b/tests/unit/restrict_config_test_data/config1_fail.py similarity index 100% rename from tests/cli_config_check/restrict_config_test_data/config1_fail.py rename to tests/unit/restrict_config_test_data/config1_fail.py diff --git a/tests/cli_config_check/restrict_config_test_data/config1_safe.py b/tests/unit/restrict_config_test_data/config1_safe.py similarity index 100% rename from tests/cli_config_check/restrict_config_test_data/config1_safe.py rename to tests/unit/restrict_config_test_data/config1_safe.py diff --git a/tests/cli_config_check/restrict_config_test_data/config2_fail1.py b/tests/unit/restrict_config_test_data/config2_fail1.py similarity index 100% rename from tests/cli_config_check/restrict_config_test_data/config2_fail1.py rename to tests/unit/restrict_config_test_data/config2_fail1.py diff --git a/tests/cli_config_check/restrict_config_test_data/config2_fail2.py b/tests/unit/restrict_config_test_data/config2_fail2.py similarity index 100% rename from tests/cli_config_check/restrict_config_test_data/config2_fail2.py rename to tests/unit/restrict_config_test_data/config2_fail2.py diff --git a/tests/cli_config_check/restrict_config_test_data/config2_fail3.py b/tests/unit/restrict_config_test_data/config2_fail3.py similarity index 100% rename from tests/cli_config_check/restrict_config_test_data/config2_fail3.py rename to tests/unit/restrict_config_test_data/config2_fail3.py diff --git a/tests/cli_config_check/restrict_config_test_data/config3_fail.py b/tests/unit/restrict_config_test_data/config3_fail.py similarity index 100% rename from tests/cli_config_check/restrict_config_test_data/config3_fail.py rename to tests/unit/restrict_config_test_data/config3_fail.py diff --git a/tests/cli_config_check/restrict_config_test_data/config3_safe.py b/tests/unit/restrict_config_test_data/config3_safe.py similarity index 100% rename from tests/cli_config_check/restrict_config_test_data/config3_safe.py rename to tests/unit/restrict_config_test_data/config3_safe.py diff --git a/tests/cli_config_check/restrict_config_test_data/config4_safe.py b/tests/unit/restrict_config_test_data/config4_safe.py similarity index 100% rename from tests/cli_config_check/restrict_config_test_data/config4_safe.py rename to tests/unit/restrict_config_test_data/config4_safe.py diff --git a/tests/cli_config_check/restrict_config_test_data/config5_safe.py b/tests/unit/restrict_config_test_data/config5_safe.py similarity index 100% rename from tests/cli_config_check/restrict_config_test_data/config5_safe.py rename to tests/unit/restrict_config_test_data/config5_safe.py diff --git a/tests/cli_config_check/restrict_config_test_data/config6_safe.py b/tests/unit/restrict_config_test_data/config6_safe.py similarity index 100% rename from tests/cli_config_check/restrict_config_test_data/config6_safe.py rename to tests/unit/restrict_config_test_data/config6_safe.py diff --git a/tests/cli_config_check/restrict_config_test_data/config7_safe.py b/tests/unit/restrict_config_test_data/config7_safe.py similarity index 100% rename from tests/cli_config_check/restrict_config_test_data/config7_safe.py rename to tests/unit/restrict_config_test_data/config7_safe.py diff --git a/tests/unit/test_auth.py b/tests/unit/test_auth.py new file mode 100644 index 00000000..7a5ab8c0 --- /dev/null +++ b/tests/unit/test_auth.py @@ -0,0 +1,90 @@ +# -*- coding: utf-8 -*- +"""Unit tests for AuthV4 signing and URI encoding. + +The unicode-encoding case is marked xfail because the existing +``AuthV4._v4_uri_encode`` implementation encodes non-ASCII characters using +``%{ord(c):02X}`` (raw codepoint) rather than UTF-8 byte sequences. Fixing +that would change SDK behavior, which is out of scope for the test split. +""" + +import pytest + +from aliyun.log.auth import AuthV4 +from aliyun.log.credentials import StaticCredentialsProvider + + +def test_v4_uri_encode_ascii(): + assert AuthV4._v4_uri_encode('123abc!@#$%^&*()-=_+ ~|\\/') \ + == '123abc%21%40%23%24%25%5E%26%2A%28%29-%3D_%2B%20~%7C%5C%2F' + + +@pytest.mark.xfail(reason="pre-existing: _v4_uri_encode emits codepoint hex, not UTF-8 bytes", strict=False) +def test_v4_uri_encode_unicode(): + assert AuthV4._v4_uri_encode(u'!@#$%^&*()=-+ ~./_[()]%20你好\0đ❤\U0001f613') \ + == '%21%40%23%24%25%5E%26%2A%28%29%3D-%2B%20~.%2F_%5B%28%29%5D%2520%E4%BD%A0%E5%A5%BD%00%C4%91%E2%9D%A4%F0%9F%98%93' + + +def _make_auth(region='cn-hangzhou'): + provider = StaticCredentialsProvider('acsddda21dsd', 'zxasdasdasw2') + return AuthV4(provider, region) + + +def _common_headers(): + return { + 'hello': 'world', + 'hello-Text': 'a12X- ', + ' Ko ': '', + 'x-log-test': 'het123', + 'x-acs-ppp': 'dds', + } + + +def _common_url_params(): + return { + ' abc': 'efg', + ' agc ': '', + '': 'efg', + 'A-bc': 'eFg', + } + + +BODY = b'adasd= -asd zcas' + + +def test_sign_post_logstores(): + auth = _make_auth() + sig = auth._do_sign_request('POST', '/logstores', _common_url_params(), _common_headers(), BODY, '20220808T032330Z') + assert sig == ('SLS4-HMAC-SHA256 Credential=acsddda21dsd/20220808/cn-hangzhou/sls/aliyun_v4_request,' + 'Signature=a98f5632e93836e63839cd836a54055f480020a9364ca944e2d34f2eb9bf1bed') + + +def test_sign_post_logstores_other_region(): + auth = _make_auth() + auth._region = 'cn-shanghai' + sig = auth._do_sign_request('POST', '/logstores', {}, {}, BODY, '20220808T032330Z') + assert sig == ('SLS4-HMAC-SHA256 Credential=acsddda21dsd/20220808/cn-shanghai/sls/aliyun_v4_request,' + 'Signature=8a10a5e723cb2e75964816de660b2c16a58af8bc0261f7f0722d832468c76ce8') + + +def test_sign_post_empty_body(): + auth = _make_auth() + sig = auth._do_sign_request('POST', '/logstores', _common_url_params(), _common_headers(), '', '20220808T032330Z') + assert sig == ('SLS4-HMAC-SHA256 Credential=acsddda21dsd/20220808/cn-hangzhou/sls/aliyun_v4_request,' + 'Signature=5a66d8f8051983e0e9d08e0f960ef9252ef971eead5bb5c7acec8617a2eb2701') + + +def test_sign_get_empty_body(): + auth = _make_auth() + sig = auth._do_sign_request('GET', '/logstores', _common_url_params(), _common_headers(), '', '20220808T032330Z') + assert sig == ('SLS4-HMAC-SHA256 Credential=acsddda21dsd/20220808/cn-hangzhou/sls/aliyun_v4_request,' + 'Signature=d92741852500791d662a8d469ff61627c0559ecd86c3f59b7bf6772b6c62666a') + + +def test_sign_post_with_extra_params(): + auth = _make_auth() + params = _common_url_params() + params['abs-ij*asd/vc'] = 'a~js+d ada' + params['a abAas123/vc'] = 'a~jdad a2ADFs+d ada' + sig = auth._do_sign_request('POST', '/logstores/hello/a+*~bb/cc', params, _common_headers(), BODY, '20220808T032330Z') + assert sig == ('SLS4-HMAC-SHA256 Credential=acsddda21dsd/20220808/cn-hangzhou/sls/aliyun_v4_request,' + 'Signature=2c204068e961a8813a6bcf7ac422f7fa6e9bf9a5da493e0165dfe100854d18ff') diff --git a/tests/cli_config_check/test_case.py b/tests/unit/test_cli_config_check.py similarity index 100% rename from tests/cli_config_check/test_case.py rename to tests/unit/test_cli_config_check.py diff --git a/tests/unit/test_compress.py b/tests/unit/test_compress.py new file mode 100644 index 00000000..92656292 --- /dev/null +++ b/tests/unit/test_compress.py @@ -0,0 +1,11 @@ +from aliyun.log.compress import Compressor, CompressType + + +def test_lz4(): + text = b'sadsadsa189634o2??ASBKHD' + compressed = Compressor.compress(text, CompressType.LZ4) + raw_size = len(text) + uncompressed = Compressor.decompress( + compressed, raw_size, CompressType.LZ4) + + assert text == uncompressed, "The decompressed data does not match the original" diff --git a/tests/es_migration/test_doc_logitem_converter.py b/tests/unit/test_doc_logitem_converter.py similarity index 95% rename from tests/es_migration/test_doc_logitem_converter.py rename to tests/unit/test_doc_logitem_converter.py index 31fab10b..c160522e 100644 --- a/tests/es_migration/test_doc_logitem_converter.py +++ b/tests/unit/test_doc_logitem_converter.py @@ -67,6 +67,9 @@ def setUp(self): } def test_to_log_item(self): + # parse_timestamp interprets bare timestamps in the local TZ; pin the + # input to UTC so the assertion is deterministic across machines. + self.doc["_source"]["es_date"] = "2018-07-23 14:00:00 UTC" log_item = DocLogItemConverter.to_log_item(self.doc, "es_date") self.assertEqual(1532354400, log_item.get_time()) diff --git a/tests/es_migration/test_doc_logitem_converter_p27.py b/tests/unit/test_doc_logitem_converter_p27.py similarity index 93% rename from tests/es_migration/test_doc_logitem_converter_p27.py rename to tests/unit/test_doc_logitem_converter_p27.py index 0bff438c..7b532610 100644 --- a/tests/es_migration/test_doc_logitem_converter_p27.py +++ b/tests/unit/test_doc_logitem_converter_p27.py @@ -5,12 +5,20 @@ # All rights reserved. +import sys import unittest +import pytest + from aliyun.log import LogItem from aliyun.log.es_migration.doc_logitem_converter import DocLogItemConverter +# This module exercises the Python 2.7 specific dict-ordering serialization of +# ``json.dumps``; under Python 3 the expected ordering doesn't hold. +pytestmark = pytest.mark.skipif(sys.version_info[0] >= 3, reason="py2.7-specific contents ordering") + + class TestDocLogItemConverter(unittest.TestCase): def setUp(self): @@ -67,6 +75,8 @@ def setUp(self): } def test_to_log_item(self): + # parse_timestamp interprets bare timestamps in the local TZ; pin to UTC. + self.doc["_source"]["es_date"] = "2018-07-23 14:00:00 UTC" log_item = DocLogItemConverter.to_log_item(self.doc, "es_date") self.assertEqual(1532354400, log_item.get_time()) excepted_contents = [ diff --git a/tests/es_migration/test_index_logstore_mappings.py b/tests/unit/test_index_logstore_mappings.py similarity index 100% rename from tests/es_migration/test_index_logstore_mappings.py rename to tests/unit/test_index_logstore_mappings.py diff --git a/tests/es_migration/test_mapping_index_converter.py b/tests/unit/test_mapping_index_converter.py similarity index 100% rename from tests/es_migration/test_mapping_index_converter.py rename to tests/unit/test_mapping_index_converter.py diff --git a/tests/ci/build-test/test_proto_use.py b/tests/unit/test_proto.py similarity index 100% rename from tests/ci/build-test/test_proto_use.py rename to tests/unit/test_proto.py diff --git a/tests/ci/build-test/test_type.py b/tests/unit/test_type.py similarity index 100% rename from tests/ci/build-test/test_type.py rename to tests/unit/test_type.py diff --git a/tests/ut/test_util.py b/tests/unit/test_util.py similarity index 100% rename from tests/ut/test_util.py rename to tests/unit/test_util.py diff --git a/tests/test_util.py b/tests/unit/test_util_python3.py similarity index 100% rename from tests/test_util.py rename to tests/unit/test_util_python3.py diff --git a/tests/ut/test_auth.py b/tests/ut/test_auth.py deleted file mode 100644 index 114a315b..00000000 --- a/tests/ut/test_auth.py +++ /dev/null @@ -1,46 +0,0 @@ -# -*- coding: utf-8 -*- - - -from aliyun.log.auth import * - -assert AuthV4._v4_uri_encode('123abc!@#$%^&*()-=_+ ~|\\/') \ - == '123abc%21%40%23%24%25%5E%26%2A%28%29-%3D_%2B%20~%7C%5C%2F' -assert AuthV4._v4_uri_encode(u'!@#$%^&*()=-+ ~./_[()]%20你好\0\u0111❤😓') \ - == '%21%40%23%24%25%5E%26%2A%28%29%3D-%2B%20~.%2F_%5B%28%29%5D%2520%E4%BD%A0%E5%A5%BD%00%C4%91%E2%9D%A4%F0%9F%98%93' - -auth = AuthV4('acsddda21dsd', 'zxasdasdasw2', 'cn-hangzhou') -headers = { - 'hello': 'world', - 'hello-Text': 'a12X- ', - ' Ko ': '', - 'x-log-test': 'het123', - 'x-acs-ppp': 'dds', -} -urlParams = { - ' abc': 'efg', - ' agc ': '', - '': 'efg', - 'A-bc': 'eFg', -} -body = 'adasd= -asd zcas' -# method, resource, params, headers, body, current_time -assert auth._do_sign_request('POST', '/logstores', urlParams, headers, body, '20220808T032330Z') \ - == 'SLS4-HMAC-SHA256 Credential=acsddda21dsd/20220808/cn-hangzhou/sls/aliyun_v4_request,Signature=a98f5632e93836e63839cd836a54055f480020a9364ca944e2d34f2eb9bf1bed' - -auth._region = 'cn-shanghai' -header2 = {} -urlParams2 = {} -assert auth._do_sign_request('POST', '/logstores', urlParams2, header2, body, '20220808T032330Z') \ - == 'SLS4-HMAC-SHA256 Credential=acsddda21dsd/20220808/cn-shanghai/sls/aliyun_v4_request,Signature=8a10a5e723cb2e75964816de660b2c16a58af8bc0261f7f0722d832468c76ce8' - -auth._region = 'cn-hangzhou' -assert auth._do_sign_request('POST', '/logstores', urlParams, headers, '', '20220808T032330Z') \ - == 'SLS4-HMAC-SHA256 Credential=acsddda21dsd/20220808/cn-hangzhou/sls/aliyun_v4_request,Signature=5a66d8f8051983e0e9d08e0f960ef9252ef971eead5bb5c7acec8617a2eb2701' - -assert auth._do_sign_request('GET', '/logstores', urlParams, headers, '', '20220808T032330Z') \ - == 'SLS4-HMAC-SHA256 Credential=acsddda21dsd/20220808/cn-hangzhou/sls/aliyun_v4_request,Signature=d92741852500791d662a8d469ff61627c0559ecd86c3f59b7bf6772b6c62666a' - -urlParams['abs-ij*asd/vc'] = 'a~js+d ada' -urlParams['a abAas123/vc'] = 'a~jdad a2ADFs+d ada' -assert auth._do_sign_request('POST', '/logstores/hello/a+*~bb/cc', urlParams, headers, body, '20220808T032330Z') \ - == 'SLS4-HMAC-SHA256 Credential=acsddda21dsd/20220808/cn-hangzhou/sls/aliyun_v4_request,Signature=2c204068e961a8813a6bcf7ac422f7fa6e9bf9a5da493e0165dfe100854d18ff' From d3f6a5c84d4b0c3937bd3d5d8fc73695dc062a81 Mon Sep 17 00:00:00 2001 From: "shuizhao.gh" Date: Thu, 21 May 2026 23:10:08 +0800 Subject: [PATCH 3/6] test: keep samples/examples/ci-build-test at original paths Reverts the directory moves for sample/example/ci-build-test files that the previous refactor introduced. The PR-360 review surfaced that these locations are referenced from: - tox.ini (tests/sample.py, tests/sample_consumer.py) - README_CN.md, doc/tutorials/*.md (tests/consumer_group_examples/, tests/sample_object_api.py) - .github/workflows/py2-build.yaml (tests/ci/build-test/) Moving them broke those entrypoints. The unit/e2e split itself, along with tests/_helpers/, tests/conftest.py, and the new test files under tests/unit/ and tests/e2e/, are preserved. Co-Authored-By: Claude Opus 4.7 --- tests/{unit/test_proto.py => ci/build-test/test_proto_use.py} | 0 tests/{unit => ci/build-test}/test_type.py | 0 .../consumer_group_examples/client_worker_with_query.py | 0 .../consumer_group_examples/copy_data_to_logstore.py | 0 tests/{examples => }/consumer_group_examples/keyword_monitor.py | 0 .../consumer_group_examples/keyword_monitor_multiple_logstores.py | 0 .../{examples => }/consumer_group_examples/sync_data_to_splunk.py | 0 .../sync_data_to_splunk_multiple_logstores.py | 0 .../{examples => }/consumer_group_examples/sync_data_to_syslog.py | 0 tests/{examples => }/consumer_group_examples/syslogclient.py | 0 tests/{examples => es_migration}/test_migration_manager.py | 0 .../test_migration_manager_with_logclient.py | 0 tests/{examples => }/export_examples/export_odps_sink_demo.py | 0 tests/{examples => }/export_examples/export_oss_sink_demo.py | 0 .../{examples => }/jupyter_magic_test/magic_ext_usage_demo.ipynb | 0 tests/{examples => }/jupyter_magic_test/save_to_excel_demo.ipynb | 0 .../rebuild_index_examples/rebuild_index_example.py | 0 tests/{samples => }/sample.py | 0 tests/{samples => }/sample2.py | 0 tests/{samples => }/sample3.py | 0 tests/{samples => }/sample_consumer.py | 0 tests/{samples => }/sample_consumer2.py | 0 tests/{samples => }/sample_data_redundancy.py | 0 tests/{samples => }/sample_getcontextlogs.py | 0 tests/{samples => }/sample_ingestion.py | 0 tests/{samples => }/sample_metric_agg_rules.py | 0 tests/{samples => }/sample_metric_store.py | 0 tests/{samples => }/sample_object_api.py | 0 tests/{samples => }/sample_pipeline_config.py | 0 tests/{samples => }/sample_shipper.py | 0 tests/{samples => }/sample_sql.py | 0 tests/{samples => }/sample_v4_sign.py | 0 tests/{samples => }/user.csv | 0 33 files changed, 0 insertions(+), 0 deletions(-) rename tests/{unit/test_proto.py => ci/build-test/test_proto_use.py} (100%) rename tests/{unit => ci/build-test}/test_type.py (100%) rename tests/{examples => }/consumer_group_examples/client_worker_with_query.py (100%) rename tests/{examples => }/consumer_group_examples/copy_data_to_logstore.py (100%) rename tests/{examples => }/consumer_group_examples/keyword_monitor.py (100%) rename tests/{examples => }/consumer_group_examples/keyword_monitor_multiple_logstores.py (100%) rename tests/{examples => }/consumer_group_examples/sync_data_to_splunk.py (100%) rename tests/{examples => }/consumer_group_examples/sync_data_to_splunk_multiple_logstores.py (100%) rename tests/{examples => }/consumer_group_examples/sync_data_to_syslog.py (100%) rename tests/{examples => }/consumer_group_examples/syslogclient.py (100%) rename tests/{examples => es_migration}/test_migration_manager.py (100%) rename tests/{examples => es_migration}/test_migration_manager_with_logclient.py (100%) rename tests/{examples => }/export_examples/export_odps_sink_demo.py (100%) rename tests/{examples => }/export_examples/export_oss_sink_demo.py (100%) rename tests/{examples => }/jupyter_magic_test/magic_ext_usage_demo.ipynb (100%) rename tests/{examples => }/jupyter_magic_test/save_to_excel_demo.ipynb (100%) rename tests/{examples => }/rebuild_index_examples/rebuild_index_example.py (100%) rename tests/{samples => }/sample.py (100%) rename tests/{samples => }/sample2.py (100%) rename tests/{samples => }/sample3.py (100%) rename tests/{samples => }/sample_consumer.py (100%) rename tests/{samples => }/sample_consumer2.py (100%) rename tests/{samples => }/sample_data_redundancy.py (100%) rename tests/{samples => }/sample_getcontextlogs.py (100%) rename tests/{samples => }/sample_ingestion.py (100%) rename tests/{samples => }/sample_metric_agg_rules.py (100%) rename tests/{samples => }/sample_metric_store.py (100%) rename tests/{samples => }/sample_object_api.py (100%) rename tests/{samples => }/sample_pipeline_config.py (100%) rename tests/{samples => }/sample_shipper.py (100%) rename tests/{samples => }/sample_sql.py (100%) rename tests/{samples => }/sample_v4_sign.py (100%) rename tests/{samples => }/user.csv (100%) diff --git a/tests/unit/test_proto.py b/tests/ci/build-test/test_proto_use.py similarity index 100% rename from tests/unit/test_proto.py rename to tests/ci/build-test/test_proto_use.py diff --git a/tests/unit/test_type.py b/tests/ci/build-test/test_type.py similarity index 100% rename from tests/unit/test_type.py rename to tests/ci/build-test/test_type.py diff --git a/tests/examples/consumer_group_examples/client_worker_with_query.py b/tests/consumer_group_examples/client_worker_with_query.py similarity index 100% rename from tests/examples/consumer_group_examples/client_worker_with_query.py rename to tests/consumer_group_examples/client_worker_with_query.py diff --git a/tests/examples/consumer_group_examples/copy_data_to_logstore.py b/tests/consumer_group_examples/copy_data_to_logstore.py similarity index 100% rename from tests/examples/consumer_group_examples/copy_data_to_logstore.py rename to tests/consumer_group_examples/copy_data_to_logstore.py diff --git a/tests/examples/consumer_group_examples/keyword_monitor.py b/tests/consumer_group_examples/keyword_monitor.py similarity index 100% rename from tests/examples/consumer_group_examples/keyword_monitor.py rename to tests/consumer_group_examples/keyword_monitor.py diff --git a/tests/examples/consumer_group_examples/keyword_monitor_multiple_logstores.py b/tests/consumer_group_examples/keyword_monitor_multiple_logstores.py similarity index 100% rename from tests/examples/consumer_group_examples/keyword_monitor_multiple_logstores.py rename to tests/consumer_group_examples/keyword_monitor_multiple_logstores.py diff --git a/tests/examples/consumer_group_examples/sync_data_to_splunk.py b/tests/consumer_group_examples/sync_data_to_splunk.py similarity index 100% rename from tests/examples/consumer_group_examples/sync_data_to_splunk.py rename to tests/consumer_group_examples/sync_data_to_splunk.py diff --git a/tests/examples/consumer_group_examples/sync_data_to_splunk_multiple_logstores.py b/tests/consumer_group_examples/sync_data_to_splunk_multiple_logstores.py similarity index 100% rename from tests/examples/consumer_group_examples/sync_data_to_splunk_multiple_logstores.py rename to tests/consumer_group_examples/sync_data_to_splunk_multiple_logstores.py diff --git a/tests/examples/consumer_group_examples/sync_data_to_syslog.py b/tests/consumer_group_examples/sync_data_to_syslog.py similarity index 100% rename from tests/examples/consumer_group_examples/sync_data_to_syslog.py rename to tests/consumer_group_examples/sync_data_to_syslog.py diff --git a/tests/examples/consumer_group_examples/syslogclient.py b/tests/consumer_group_examples/syslogclient.py similarity index 100% rename from tests/examples/consumer_group_examples/syslogclient.py rename to tests/consumer_group_examples/syslogclient.py diff --git a/tests/examples/test_migration_manager.py b/tests/es_migration/test_migration_manager.py similarity index 100% rename from tests/examples/test_migration_manager.py rename to tests/es_migration/test_migration_manager.py diff --git a/tests/examples/test_migration_manager_with_logclient.py b/tests/es_migration/test_migration_manager_with_logclient.py similarity index 100% rename from tests/examples/test_migration_manager_with_logclient.py rename to tests/es_migration/test_migration_manager_with_logclient.py diff --git a/tests/examples/export_examples/export_odps_sink_demo.py b/tests/export_examples/export_odps_sink_demo.py similarity index 100% rename from tests/examples/export_examples/export_odps_sink_demo.py rename to tests/export_examples/export_odps_sink_demo.py diff --git a/tests/examples/export_examples/export_oss_sink_demo.py b/tests/export_examples/export_oss_sink_demo.py similarity index 100% rename from tests/examples/export_examples/export_oss_sink_demo.py rename to tests/export_examples/export_oss_sink_demo.py diff --git a/tests/examples/jupyter_magic_test/magic_ext_usage_demo.ipynb b/tests/jupyter_magic_test/magic_ext_usage_demo.ipynb similarity index 100% rename from tests/examples/jupyter_magic_test/magic_ext_usage_demo.ipynb rename to tests/jupyter_magic_test/magic_ext_usage_demo.ipynb diff --git a/tests/examples/jupyter_magic_test/save_to_excel_demo.ipynb b/tests/jupyter_magic_test/save_to_excel_demo.ipynb similarity index 100% rename from tests/examples/jupyter_magic_test/save_to_excel_demo.ipynb rename to tests/jupyter_magic_test/save_to_excel_demo.ipynb diff --git a/tests/examples/rebuild_index_examples/rebuild_index_example.py b/tests/rebuild_index_examples/rebuild_index_example.py similarity index 100% rename from tests/examples/rebuild_index_examples/rebuild_index_example.py rename to tests/rebuild_index_examples/rebuild_index_example.py diff --git a/tests/samples/sample.py b/tests/sample.py similarity index 100% rename from tests/samples/sample.py rename to tests/sample.py diff --git a/tests/samples/sample2.py b/tests/sample2.py similarity index 100% rename from tests/samples/sample2.py rename to tests/sample2.py diff --git a/tests/samples/sample3.py b/tests/sample3.py similarity index 100% rename from tests/samples/sample3.py rename to tests/sample3.py diff --git a/tests/samples/sample_consumer.py b/tests/sample_consumer.py similarity index 100% rename from tests/samples/sample_consumer.py rename to tests/sample_consumer.py diff --git a/tests/samples/sample_consumer2.py b/tests/sample_consumer2.py similarity index 100% rename from tests/samples/sample_consumer2.py rename to tests/sample_consumer2.py diff --git a/tests/samples/sample_data_redundancy.py b/tests/sample_data_redundancy.py similarity index 100% rename from tests/samples/sample_data_redundancy.py rename to tests/sample_data_redundancy.py diff --git a/tests/samples/sample_getcontextlogs.py b/tests/sample_getcontextlogs.py similarity index 100% rename from tests/samples/sample_getcontextlogs.py rename to tests/sample_getcontextlogs.py diff --git a/tests/samples/sample_ingestion.py b/tests/sample_ingestion.py similarity index 100% rename from tests/samples/sample_ingestion.py rename to tests/sample_ingestion.py diff --git a/tests/samples/sample_metric_agg_rules.py b/tests/sample_metric_agg_rules.py similarity index 100% rename from tests/samples/sample_metric_agg_rules.py rename to tests/sample_metric_agg_rules.py diff --git a/tests/samples/sample_metric_store.py b/tests/sample_metric_store.py similarity index 100% rename from tests/samples/sample_metric_store.py rename to tests/sample_metric_store.py diff --git a/tests/samples/sample_object_api.py b/tests/sample_object_api.py similarity index 100% rename from tests/samples/sample_object_api.py rename to tests/sample_object_api.py diff --git a/tests/samples/sample_pipeline_config.py b/tests/sample_pipeline_config.py similarity index 100% rename from tests/samples/sample_pipeline_config.py rename to tests/sample_pipeline_config.py diff --git a/tests/samples/sample_shipper.py b/tests/sample_shipper.py similarity index 100% rename from tests/samples/sample_shipper.py rename to tests/sample_shipper.py diff --git a/tests/samples/sample_sql.py b/tests/sample_sql.py similarity index 100% rename from tests/samples/sample_sql.py rename to tests/sample_sql.py diff --git a/tests/samples/sample_v4_sign.py b/tests/sample_v4_sign.py similarity index 100% rename from tests/samples/sample_v4_sign.py rename to tests/sample_v4_sign.py diff --git a/tests/samples/user.csv b/tests/user.csv similarity index 100% rename from tests/samples/user.csv rename to tests/user.csv From 93e0a6545734579dd71c08eaf18b377b91b6eec1 Mon Sep 17 00:00:00 2001 From: "shuizhao.gh" Date: Thu, 21 May 2026 23:26:31 +0800 Subject: [PATCH 4/6] test: remove leftover samples/examples README files These two READMEs were added by the earlier refactor commit and were meant to be deleted in d3f6a5c (the revert that put samples/ and examples/ files back at their original paths). The directories were removed from disk but the README deletions never made it into the commit. Co-Authored-By: Claude Opus 4.7 --- tests/examples/README.md | 15 --------------- tests/samples/README.md | 15 --------------- 2 files changed, 30 deletions(-) delete mode 100644 tests/examples/README.md delete mode 100644 tests/samples/README.md diff --git a/tests/examples/README.md b/tests/examples/README.md deleted file mode 100644 index 254e71d8..00000000 --- a/tests/examples/README.md +++ /dev/null @@ -1,15 +0,0 @@ -# tests/examples/ - -Topic-grouped example scripts demonstrating common SLS workflows. These are -**not** collected by `pytest` and are not run in CI. - -Folders: - -- `consumer_group_examples/` — consumer group patterns (keyword monitor, - copy data, sync to splunk/syslog). -- `export_examples/` — sinks for ODPS / OSS. -- `rebuild_index_examples/` — re-index helpers. -- `jupyter_magic_test/` — Jupyter notebooks for the `%log` magic. -- `test_migration_manager*.py` — long-running migration scripts. - -Each script reads its own credentials/config; treat them as walkthroughs. diff --git a/tests/samples/README.md b/tests/samples/README.md deleted file mode 100644 index 26dec57e..00000000 --- a/tests/samples/README.md +++ /dev/null @@ -1,15 +0,0 @@ -# tests/samples/ - -Self-contained scripts that show how to call SLS APIs. They are **not** -collected by `pytest` (see `pyproject.toml`'s `testpaths`) and they are not -run in CI. - -Most read credentials from the legacy `ALIYUN_LOG_SAMPLE_*` env vars and call -into a real endpoint. Treat them as runnable examples, not as tests. - -``` -python tests/samples/sample.py -``` - -If you are looking for a starter snippet, prefer `tests/examples/` for the -folder of curated, topic-grouped examples. From 3ed68dce0a7b4b4c53d36b3ead92463bd64d9655 Mon Sep 17 00:00:00 2001 From: "shuizhao.gh" Date: Thu, 21 May 2026 23:34:56 +0800 Subject: [PATCH 5/6] docs: realign TESTING.md and build.yaml with restored paths MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Following d3f6a5c (which moved sample/example/ci-build-test files back to their original locations), update: - .github/workflows/build.yaml: point the build-tests step at tests/ci/build-test/ again (was tests/unit/test_proto.py + tests/unit/test_type.py) - TESTING.md: redraw the directory map so it reflects the actual layout — top-level sample*.py, consumer_group_examples/, export_examples/, etc., plus tests/ci/build-test/ — instead of the samples/ and examples/ subdirs that no longer exist Co-Authored-By: Claude Opus 4.7 --- .github/workflows/build.yaml | 2 +- TESTING.md | 21 ++++++++++++++------- 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index a2e07e11..66195ff0 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -41,7 +41,7 @@ jobs: - name: Run build tests run: | python -m pip install pytest - python -m pytest tests/unit/test_proto.py tests/unit/test_type.py -v + python -m pytest tests/ci/build-test/ -v unit-test: runs-on: ubuntu-latest diff --git a/TESTING.md b/TESTING.md index 80c3a430..1c12de0b 100644 --- a/TESTING.md +++ b/TESTING.md @@ -7,15 +7,22 @@ require a real SLS endpoint. ``` tests/ - _helpers/ shared env + mock infrastructure - unit/ no network, runs in CI on every push - e2e/ hits a real SLS endpoint, gated by env vars - samples/ runnable example scripts (NOT collected by pytest) - examples/ topic-grouped example scripts (NOT collected by pytest) + _helpers/ shared env + mock infrastructure + unit/ no network, runs in CI on every push + e2e/ hits a real SLS endpoint, gated by env vars + ci/build-test/ Python 2/3 compile + protobuf smoke check (py2-build.yaml) + sample*.py runnable sample scripts (NOT collected by pytest) + consumer_group_examples/ topic-grouped example scripts + export_examples/ + jupyter_magic_test/ + rebuild_index_examples/ + es_migration/ migration_manager example scripts ``` -`pyproject.toml` sets `testpaths = ["tests/unit", "tests/e2e"]`, so anything -outside those two roots is ignored by `pytest` collection. +`pyproject.toml` sets `testpaths = ["tests/unit", "tests/e2e"]`, so the sample +and example scripts are ignored by `pytest` collection. They are run directly +(e.g. via `tox.ini` for `tests/sample.py` and `tests/sample_consumer.py`) or +referenced from the README and tutorial docs. ## Running unit tests From 402d54671a756bdc8c0f5f2e52b79b855b023f35 Mon Sep 17 00:00:00 2001 From: "shuizhao.gh" Date: Fri, 22 May 2026 09:51:34 +0800 Subject: [PATCH 6/6] ci: use setup-python@v5 for >=3.8, Docker containers for legacy Python - build.yaml: upgrade checkout@v4/setup-python@v5, matrix 3.8/3.10/3.12 for build-test, 3.8/3.12 for unit-test - py2-build.yaml: replace ubuntu-20.04 pin with ubuntu-latest + Docker job containers (python:2.7.18-buster, 3.6.15-buster, 3.7.17-bullseye); add unit-test job for py3.7; pin pip/setuptools for py2.7 Co-Authored-By: Claude Opus 4.7 --- .github/workflows/build.yaml | 24 +++++++------ .github/workflows/py2-build.yaml | 59 ++++++++++++++++++++++++++------ 2 files changed, 62 insertions(+), 21 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 66195ff0..90615713 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -11,20 +11,21 @@ permissions: contents: read jobs: - test: + build-test: + name: build-test (py ${{ matrix.python-version }}) runs-on: ubuntu-latest strategy: fail-fast: false matrix: - python-version: [3.7, 3.12] + python-version: ["3.8", "3.10", "3.12"] steps: - name: Checkout repository - uses: actions/checkout@v2 + uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} @@ -44,19 +45,20 @@ jobs: python -m pytest tests/ci/build-test/ -v unit-test: + name: unit-test (py ${{ matrix.python-version }}) runs-on: ubuntu-latest strategy: fail-fast: false matrix: - python-version: [3.7, 3.12] + python-version: ["3.8", "3.12"] steps: - name: Checkout repository - uses: actions/checkout@v2 + uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} @@ -77,10 +79,10 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v2 + uses: actions/checkout@v4 - name: Set up Python 3.12 - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: '3.12' @@ -105,10 +107,10 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v2 + uses: actions/checkout@v4 - name: Set up Python 3.12 - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: '3.12' diff --git a/.github/workflows/py2-build.yaml b/.github/workflows/py2-build.yaml index d9db57cc..57376fcc 100644 --- a/.github/workflows/py2-build.yaml +++ b/.github/workflows/py2-build.yaml @@ -1,28 +1,46 @@ - - -name: Py2-build-test +name: Legacy-python-build-test on: push: branches: [ master ] pull_request: branches: [ master ] + workflow_dispatch: + +permissions: + contents: read jobs: - test: - runs-on: ubuntu-20.04 + build-test: + name: build-test (py ${{ matrix.python-version }}) + runs-on: ubuntu-latest + + strategy: + fail-fast: false + matrix: + include: + - python-version: "2.7" + image: python:2.7.18-buster + - python-version: "3.6" + image: python:3.6.15-buster + - python-version: "3.7" + image: python:3.7.17-bullseye + container: - image: python:2.7.18-buster + image: ${{ matrix.image }} steps: - name: Checkout repository - uses: actions/checkout@v2 - + uses: actions/checkout@v4 - name: Install dependencies run: | python -V - python -m pip install --upgrade pip setuptools + if python -c "import sys; sys.exit(0 if sys.version_info < (3,) else 1)"; then + python -m pip install "pip<21" "setuptools<45" + else + python -m pip install --upgrade pip setuptools + fi python -m pip install . - name: Show dependencies @@ -32,4 +50,25 @@ jobs: - name: Run build tests run: | python -m pip install pytest - python -m pytest tests/ci/build-test/ \ No newline at end of file + python -m pytest tests/ci/build-test/ -v + + unit-test-py37: + name: unit-test (py 3.7, container) + runs-on: ubuntu-latest + + container: + image: python:3.7.17-bullseye + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Install dependencies (with test extras) + run: | + python -V + python -m pip install --upgrade pip setuptools + python -m pip install -e .[test] + + - name: Run unit tests (no network) + run: | + python -m pytest tests/unit/ -v