Skip to content

Commit 7afdef3

Browse files
rbiseck3rbiseck3
authored
feat/Leverage uvicorn trace level to log body (#22)
1 parent 50500e5 commit 7afdef3

File tree

12 files changed

+116
-79
lines changed

12 files changed

+116
-79
lines changed

‎.github/workflows/release.yml‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,8 +4,6 @@ on:
44
release:
55
types:
66
- published
7-
- created
8-
- released
97

108
env:
119
PYTHON_VERSION: "3.10"

‎CHANGELOG.md‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,12 @@
1+
## 0.0.5
2+
3+
* **Improve logging to hide body in case of sensitive data unless TRACE level**
4+
15
## 0.0.4
26

37
### Fixes
48

5-
* Do not block event loop when running plugin code
9+
* **Do not block event loop when running plugin code**
610

711
## 0.0.3
812

‎requirements/cli.txt‎

Lines changed: 32 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -2,17 +2,17 @@
22
# This file is autogenerated by pip-compile with Python 3.10
33
# by the following command:
44
#
5-
# pip-compile requirements//cli.in
5+
# pip-compile ./cli.in
66
#
77
annotated-types==0.7.0
88
# via pydantic
9-
anyio==4.4.0
9+
anyio==4.6.0
1010
# via starlette
1111
asgiref==3.8.1
1212
# via opentelemetry-instrumentation-asgi
1313
click==8.1.7
1414
# via
15-
# -r requirements//cli.in
15+
# -r ./cli.in
1616
# unstructured-ingest
1717
# uvicorn
1818
dataclasses-json==0.6.7
@@ -24,86 +24,86 @@ deprecated==1.2.14
2424
# opentelemetry-semantic-conventions
2525
exceptiongroup==1.2.2
2626
# via anyio
27-
fastapi==0.112.2
28-
# via -r requirements//cli.in
27+
fastapi==0.115.0
28+
# via -r ./cli.in
2929
googleapis-common-protos==1.65.0
3030
# via opentelemetry-exporter-otlp-proto-grpc
31-
grpcio==1.66.0
31+
grpcio==1.66.1
3232
# via opentelemetry-exporter-otlp-proto-grpc
3333
h11==0.14.0
3434
# via uvicorn
35-
idna==3.8
35+
idna==3.10
3636
# via anyio
37-
importlib-metadata==8.0.0
37+
importlib-metadata==8.4.0
3838
# via opentelemetry-api
3939
marshmallow==3.22.0
4040
# via dataclasses-json
4141
mypy-extensions==1.0.0
4242
# via typing-inspect
43-
numpy==2.1.0
43+
numpy==2.1.1
4444
# via pandas
45-
opentelemetry-api==1.26.0
45+
opentelemetry-api==1.27.0
4646
# via
4747
# opentelemetry-exporter-otlp-proto-grpc
4848
# opentelemetry-instrumentation
4949
# opentelemetry-instrumentation-asgi
5050
# opentelemetry-instrumentation-fastapi
5151
# opentelemetry-sdk
5252
# opentelemetry-semantic-conventions
53-
opentelemetry-exporter-otlp-proto-common==1.26.0
53+
opentelemetry-exporter-otlp-proto-common==1.27.0
5454
# via opentelemetry-exporter-otlp-proto-grpc
55-
opentelemetry-exporter-otlp-proto-grpc==1.26.0
56-
# via -r requirements//cli.in
57-
opentelemetry-instrumentation==0.47b0
55+
opentelemetry-exporter-otlp-proto-grpc==1.27.0
56+
# via -r ./cli.in
57+
opentelemetry-instrumentation==0.48b0
5858
# via
5959
# opentelemetry-instrumentation-asgi
6060
# opentelemetry-instrumentation-fastapi
61-
opentelemetry-instrumentation-asgi==0.47b0
61+
opentelemetry-instrumentation-asgi==0.48b0
6262
# via opentelemetry-instrumentation-fastapi
63-
opentelemetry-instrumentation-fastapi==0.47b0
64-
# via -r requirements//cli.in
65-
opentelemetry-proto==1.26.0
63+
opentelemetry-instrumentation-fastapi==0.48b0
64+
# via -r ./cli.in
65+
opentelemetry-proto==1.27.0
6666
# via
6767
# opentelemetry-exporter-otlp-proto-common
6868
# opentelemetry-exporter-otlp-proto-grpc
69-
opentelemetry-sdk==1.26.0
69+
opentelemetry-sdk==1.27.0
7070
# via
7171
# opentelemetry-exporter-otlp-proto-grpc
7272
# unstructured-ingest
73-
opentelemetry-semantic-conventions==0.47b0
73+
opentelemetry-semantic-conventions==0.48b0
7474
# via
7575
# opentelemetry-instrumentation-asgi
7676
# opentelemetry-instrumentation-fastapi
7777
# opentelemetry-sdk
78-
opentelemetry-util-http==0.47b0
78+
opentelemetry-util-http==0.48b0
7979
# via
8080
# opentelemetry-instrumentation-asgi
8181
# opentelemetry-instrumentation-fastapi
8282
packaging==24.1
8383
# via marshmallow
84-
pandas==2.2.2
84+
pandas==2.2.3
8585
# via unstructured-ingest
86-
protobuf==4.25.4
86+
protobuf==4.25.5
8787
# via
8888
# googleapis-common-protos
8989
# opentelemetry-proto
90-
pydantic==2.8.2
90+
pydantic==2.9.2
9191
# via
9292
# fastapi
9393
# unstructured-ingest
94-
pydantic-core==2.20.1
94+
pydantic-core==2.23.4
9595
# via pydantic
9696
python-dateutil==2.9.0.post0
9797
# via
9898
# pandas
9999
# unstructured-ingest
100-
pytz==2024.1
100+
pytz==2024.2
101101
# via pandas
102102
six==1.16.0
103103
# via python-dateutil
104104
sniffio==1.3.1
105105
# via anyio
106-
starlette==0.38.2
106+
starlette==0.38.6
107107
# via fastapi
108108
tqdm==4.66.5
109109
# via unstructured-ingest
@@ -119,17 +119,17 @@ typing-extensions==4.12.2
119119
# uvicorn
120120
typing-inspect==0.9.0
121121
# via dataclasses-json
122-
tzdata==2024.1
122+
tzdata==2024.2
123123
# via pandas
124-
unstructured-ingest==0.0.8
125-
# via -r requirements//cli.in
124+
unstructured-ingest==0.0.18
125+
# via -r ./cli.in
126126
uvicorn==0.30.6
127-
# via -r requirements//cli.in
127+
# via -r ./cli.in
128128
wrapt==1.16.0
129129
# via
130130
# deprecated
131131
# opentelemetry-instrumentation
132-
zipp==3.20.1
132+
zipp==3.20.2
133133
# via importlib-metadata
134134

135135
# The following packages are considered to be unsafe in a requirements file:

‎requirements/constraints.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
unstructured-ingest==0.0.8
1+
unstructured-ingest>=0.0.18

‎requirements/lint.txt‎

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -2,24 +2,24 @@
22
# This file is autogenerated by pip-compile with Python 3.10
33
# by the following command:
44
#
5-
# pip-compile requirements//lint.in
5+
# pip-compile ./lint.in
66
#
77
autoflake==2.3.1
8-
# via -r requirements//lint.in
8+
# via -r ./lint.in
99
black==24.8.0
10-
# via -r requirements//lint.in
10+
# via -r ./lint.in
1111
click==8.1.7
1212
# via black
1313
flake8==7.1.1
1414
# via
15-
# -r requirements//lint.in
15+
# -r ./lint.in
1616
# flake8-print
1717
flake8-print==5.0.0
18-
# via -r requirements//lint.in
18+
# via -r ./lint.in
1919
mccabe==0.7.0
2020
# via flake8
2121
mypy==1.11.2
22-
# via -r requirements//lint.in
22+
# via -r ./lint.in
2323
mypy-extensions==1.0.0
2424
# via
2525
# black
@@ -28,7 +28,7 @@ packaging==24.1
2828
# via black
2929
pathspec==0.12.1
3030
# via black
31-
platformdirs==4.2.2
31+
platformdirs==4.3.6
3232
# via black
3333
pycodestyle==2.12.1
3434
# via
@@ -38,8 +38,8 @@ pyflakes==3.2.0
3838
# via
3939
# autoflake
4040
# flake8
41-
ruff==0.6.2
42-
# via -r requirements//lint.in
41+
ruff==0.6.7
42+
# via -r ./lint.in
4343
tomli==2.0.1
4444
# via
4545
# autoflake

‎requirements/release.txt‎

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -2,19 +2,19 @@
22
# This file is autogenerated by pip-compile with Python 3.10
33
# by the following command:
44
#
5-
# pip-compile requirements//release.in
5+
# pip-compile ./release.in
66
#
77
backports-tarfile==1.2.0
88
# via jaraco-context
9-
certifi==2024.7.4
9+
certifi==2024.8.30
1010
# via requests
1111
charset-normalizer==3.3.2
1212
# via requests
1313
docutils==0.21.2
1414
# via readme-renderer
15-
idna==3.8
15+
idna==3.10
1616
# via requests
17-
importlib-metadata==8.4.0
17+
importlib-metadata==8.5.0
1818
# via
1919
# keyring
2020
# twine
@@ -24,13 +24,13 @@ jaraco-context==6.0.1
2424
# via keyring
2525
jaraco-functools==4.0.2
2626
# via keyring
27-
keyring==25.3.0
27+
keyring==25.4.1
2828
# via twine
2929
markdown-it-py==3.0.0
3030
# via rich
3131
mdurl==0.1.2
3232
# via markdown-it-py
33-
more-itertools==10.4.0
33+
more-itertools==10.5.0
3434
# via
3535
# jaraco-classes
3636
# jaraco-functools
@@ -52,15 +52,15 @@ requests-toolbelt==1.0.0
5252
# via twine
5353
rfc3986==2.0.0
5454
# via twine
55-
rich==13.8.0
55+
rich==13.8.1
5656
# via twine
5757
twine==5.1.1
58-
# via -r requirements//release.in
59-
urllib3==2.2.2
58+
# via -r ./release.in
59+
urllib3==2.2.3
6060
# via
6161
# requests
6262
# twine
6363
wheel==0.44.0
64-
# via -r requirements//release.in
65-
zipp==3.20.1
64+
# via -r ./release.in
65+
zipp==3.20.2
6666
# via importlib-metadata

‎requirements/test.txt‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
# This file is autogenerated by pip-compile with Python 3.10
33
# by the following command:
44
#
5-
# pip-compile requirements//test.in
5+
# pip-compile ./test.in
66
#
77
exceptiongroup==1.2.2
88
# via pytest
@@ -12,7 +12,7 @@ packaging==24.1
1212
# via pytest
1313
pluggy==1.5.0
1414
# via pytest
15-
pytest==8.3.2
16-
# via -r requirements//test.in
15+
pytest==8.3.3
16+
# via -r ./test.in
1717
tomli==2.0.1
1818
# via pytest

‎requirements/validate.txt‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -2,17 +2,17 @@
22
# This file is autogenerated by pip-compile with Python 3.10
33
# by the following command:
44
#
5-
# pip-compile requirements//validate.in
5+
# pip-compile ./validate.in
66
#
7-
certifi==2024.7.4
7+
certifi==2024.8.30
88
# via requests
99
charset-normalizer==3.3.2
1010
# via requests
1111
click==8.1.7
12-
# via -r requirements//validate.in
13-
idna==3.8
12+
# via -r ./validate.in
13+
idna==3.10
1414
# via requests
1515
requests==2.32.3
16-
# via -r requirements//validate.in
17-
urllib3==2.2.2
16+
# via -r ./validate.in
17+
urllib3==2.2.3
1818
# via requests

‎scripts/pip-compile.sh‎

Lines changed: 19 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,18 +1,27 @@
11
#!/usr/bin/env bash
22

3+
pushd ./requirements || exit
4+
5+
find . -type f -name "*.txt" ! -name "constraints.txt" -exec rm '{}' ';'
6+
find . -type f -name "*.in" -maxdepth 1 -exec pip-compile --upgrade '{}' ';'
7+
8+
popd || exit
9+
10+
# Check python version
311
# python version must match lowest supported (3.10)
412
major=3
513
minor=10
6-
if ! python -c "import sys; assert sys.version_info.major == $major and sys.version_info.minor == $minor"; then
7-
echo "python version not equal to expected $major.$minor: $(python --version)"
14+
15+
versions=$(cat requirements/* | grep "This file is autogenerated by pip-compile with Python" | awk '{print $NF}' | sort | uniq)
16+
if [[ $(echo $versions | wc -w) -ne 1 ]]; then
17+
echo "Files generated with multiple python version: $versions"
818
exit 1
919
fi
1020

11-
while IFS= read -r -d '' file; do
12-
filename=${file%.in}
13-
txtfilename="${filename}.txt"
14-
echo "Removing $txtfilename"
15-
rm -f $txtfilename
16-
echo "Running pip-compile on file $file"
17-
pip-compile --upgrade --verbose "$file"
18-
done < <(find requirements/ -type f -name "*.in" -maxdepth 1 -print0)
21+
version_major=$(echo $versions | awk -F"." '{print $1}')
22+
version_minor=$(echo $versions | awk -F"." '{print $2}')
23+
24+
if [[ $major -ne $version_major || $minor -ne $version_minor ]]; then
25+
echo "python version not equal to expected $major.$minor: $versions"
26+
exit 1
27+
fi

‎test/test_schema.py‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -566,6 +566,10 @@ def fn(a: FileData) -> list[FileData]:
566566
"items": {"key": {"type": "string"}, "value": {}},
567567
},
568568
"reprocess": {"type": "boolean", "default": False},
569+
"local_download_path": {
570+
"anyOf": [{"type": "string"}, {"type": "null"}],
571+
"default": None,
572+
},
569573
},
570574
"required": ["identifier", "connector_type", "metadata", "additional_metadata"],
571575
}
@@ -659,6 +663,10 @@ def fn(a: FileData) -> list[FileData]:
659663
"items": {"key": {"type": "string"}, "value": {}},
660664
},
661665
"reprocess": {"type": "boolean", "default": False},
666+
"local_download_path": {
667+
"anyOf": [{"type": "string"}, {"type": "null"}],
668+
"default": None,
669+
},
662670
},
663671
"required": ["identifier", "connector_type", "metadata", "additional_metadata"],
664672
},

0 commit comments

Comments
 (0)