Skip to content
This repository was archived by the owner on Jun 3, 2025. It is now read-only.

Commit de2802b

Browse files
bfineranbfineran
authored
remove auto installs and switch to pypi forks for transformers and yolov5 (#1518) (#1554)
* remove auto installs and switch to pypi forks for transformers and yolov5 * quality * quality
1 parent aee7a0c commit de2802b

File tree

4 files changed

+26
-176
lines changed

4 files changed

+26
-176
lines changed

‎docker/Dockerfile‎

Lines changed: 5 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -95,23 +95,19 @@ ARG MODE
9595
RUN \
9696
if [ -n "$BRANCH" ] ; then \
9797
echo Installing from BRANCH && \
98-
$VENV/bin/pip install --no-cache-dir "./sparseml[onnxruntime,torchvision,ultralytics]"; \
98+
$VENV/bin/pip install --no-cache-dir "./sparseml[onnxruntime,torchvision,transformers,yolov5,ultralytics]"; \
9999
elif [ "$MODE" = "nightly" ] ; then \
100100
if [ -z $VERSION] ; then \
101-
$VENV/bin/pip install --no-cache-dir "sparseml-nightly[onnxruntime,torchvision,ultralytics]"; \
101+
$VENV/bin/pip install --no-cache-dir "sparseml-nightly[onnxruntime,torchvision,transformers,yolov5,ultralytics]"; \
102102
else \
103-
$VENV/bin/pip install --no-cache-dir "sparseml-nightly[onnxruntime,torchvision,ultralytics]==$VERSION"; \
103+
$VENV/bin/pip install --no-cache-dir "sparseml-nightly[onnxruntime,torchvision,transformers,yolov5,ultralytics]==$VERSION"; \
104104
fi; \
105105
elif [ -z $VERSION] ; then \
106-
$VENV/bin/pip install --no-cache-dir "sparseml[onnxruntime,torchvision,ultralytics]"; \
106+
$VENV/bin/pip install --no-cache-dir "sparseml[onnxruntime,torchvision,transformers,yolov5,ultralytics]"; \
107107
else \
108-
$VENV/bin/pip install --no-cache-dir "sparseml[onnxruntime,torchvision,ultralytics]==$VERSION"; \
108+
$VENV/bin/pip install --no-cache-dir "sparseml[onnxruntime,torchvision,transformers,yolov5,ultralytics]==$VERSION"; \
109109
fi;
110110

111-
RUN sparseml.transformers.question_answering --help \
112-
&& sparseml.yolov5.train --help \
113-
&& sparseml.ultralytics.train --help
114-
115111

116112
FROM cuda_builder AS container_branch_dev
117113
ARG VENV

‎setup.py‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -71,6 +71,16 @@
7171
"torchvision>=0.3.0,<=0.14",
7272
]
7373
_pytorch_vision_deps = _pytorch_deps + ["torchvision>=0.3.0,<=0.14"]
74+
_transformers_deps = _pytorch_deps + [
75+
f"{'nm-transformers' if is_release else 'nm-transformers-nightly'}"
76+
f"~={version_nm_deps}",
77+
"datasets<=1.18.4",
78+
"scikit-learn",
79+
"seqeval",
80+
]
81+
_yolov5_deps = _pytorch_vision_deps + [
82+
f"{'nm-yolov5' if is_release else 'nm-yolov5-nightly'}~={version_nm_deps}"
83+
]
7484
_tensorflow_v1_deps = ["tensorflow<2.0.0", "tensorboard<2.0.0", "tf2onnx>=1.0.0,<1.6"]
7585
_tensorflow_v1_gpu_deps = [
7686
"tensorflow-gpu<2.0.0",
@@ -132,10 +142,12 @@ def _setup_extras() -> Dict:
132142
"torch": _pytorch_deps,
133143
"torch_all": _pytorch_all_deps,
134144
"torchvision": _pytorch_vision_deps,
145+
"transformers": _transformers_deps,
135146
"tf_v1": _tensorflow_v1_deps,
136147
"tf_v1_gpu": _tensorflow_v1_gpu_deps,
137148
"tf_keras": _keras_deps,
138149
"ultralytics": _ultralytics_deps,
150+
"yolov5": _yolov5_deps,
139151
}
140152

141153

‎src/sparseml/transformers/__init__.py‎

Lines changed: 3 additions & 94 deletions
Original file line numberDiff line numberDiff line change
@@ -20,111 +20,20 @@
2020

2121
import logging as _logging
2222

23-
import pkg_resources
2423
from sparseml.analytics import sparseml_analytics as _analytics
2524

2625

2726
_analytics.send_event("python__transformers__init")
2827

29-
_EXPECTED_VERSION = "4.23.1"
30-
3128

3229
_LOGGER = _logging.getLogger(__name__)
33-
_NM_TRANSFORMERS_TAR_TEMPLATE = (
34-
"https://github.com/neuralmagic/transformers/releases/download/"
35-
"{version}/transformers-4.23.1-py3-none-any.whl"
36-
)
37-
_NM_TRANSFORMERS_NIGHTLY = _NM_TRANSFORMERS_TAR_TEMPLATE.format(version="nightly")
38-
39-
40-
def _install_transformers_and_deps():
41-
42-
import subprocess as _subprocess
43-
import sys as _sys
44-
45-
import sparseml as _sparseml
46-
47-
nm_transformers_release = (
48-
"nightly" if not _sparseml.is_release else f"v{_sparseml.version_major_minor}"
49-
)
50-
transformers_requirement = _NM_TRANSFORMERS_TAR_TEMPLATE.format(
51-
version=nm_transformers_release
52-
)
53-
try:
54-
_subprocess.check_call(
55-
[
56-
_sys.executable,
57-
"-m",
58-
"pip",
59-
"install",
60-
transformers_requirement,
61-
"datasets<=1.18.4",
62-
"scikit-learn",
63-
"seqeval",
64-
]
65-
)
66-
67-
import transformers as _transformers
68-
69-
_LOGGER.info("sparseml-transformers and dependencies successfully installed")
70-
except Exception:
71-
raise ValueError(
72-
"Unable to install and import sparseml-transformers dependencies check "
73-
"that transformers is installed, if not, install via "
74-
f"`pip install {_NM_TRANSFORMERS_NIGHTLY}`"
75-
)
7630

7731

7832
def _check_transformers_install():
79-
transformers_version = next(
80-
(
81-
pkg.version
82-
for pkg in pkg_resources.working_set
83-
if pkg.project_name.lower() == "transformers"
84-
),
85-
None,
86-
)
87-
88-
# Either no transformers install is found or wrong version installed
89-
if transformers_version != _EXPECTED_VERSION:
90-
import os
91-
92-
if os.getenv("NM_NO_AUTOINSTALL_TRANSFORMERS", False):
93-
_LOGGER.warning(
94-
"Unable to import, skipping auto installation "
95-
"due to NM_NO_AUTOINSTALL_TRANSFORMERS"
96-
)
97-
# skip any further checks
98-
return
99-
else:
100-
_LOGGER.warning(
101-
f"sparseml-transformers v{_EXPECTED_VERSION} installation not "
102-
f"detected. Installing sparseml-transformers v{_EXPECTED_VERSION} "
103-
"dependencies if transformers is already installed in the "
104-
"environment, it will be overwritten. Set environment variable "
105-
"NM_NO_AUTOINSTALL_TRANSFORMERS to disable"
106-
)
107-
_install_transformers_and_deps()
108-
109-
else:
110-
import transformers as _transformers
111-
112-
# Edge case where user has expected version of transformers installed, but
113-
# not the nm integrated one
114-
if not _transformers.NM_INTEGRATED:
115-
_install_transformers_and_deps()
116-
raise RuntimeError(
117-
"Installed transformers package has been overwritten with "
118-
"sparseml-transformers. Stopping process as this is likely to cause "
119-
"import issues. Please re-run command"
120-
)
121-
122-
# re check import after potential install
123-
try:
124-
import transformers as _transformers
33+
# check for NM integration in transformers version
34+
import transformers as _transformers
12535

126-
assert _transformers.NM_INTEGRATED
127-
except Exception:
36+
if not _transformers.NM_INTEGRATED:
12837
_LOGGER.warning(
12938
"the neuralmagic fork of transformers may not be installed. it can be "
13039
"installed via "

‎src/sparseml/yolov5/__init__.py‎

Lines changed: 6 additions & 73 deletions
Original file line numberDiff line numberDiff line change
@@ -21,90 +21,23 @@
2121

2222
from sparseml.analytics import sparseml_analytics as _analytics
2323

24-
from .helpers import *
25-
2624

2725
_analytics.send_event("python__yolov5__init")
2826

29-
try:
30-
import yolov5 as _yolov5
31-
32-
_yolov5_import_error = None
33-
except Exception as _yolov5_import_err:
34-
_yolov5_import_error = _yolov5_import_err
35-
3627
_LOGGER = _logging.getLogger(__name__)
37-
_NM_YOLOV5_TAR_TEMPLATE = (
38-
"https://github.com/neuralmagic/yolov5/releases/download/"
39-
"{version}/yolov5-6.2.0-py3-none-any.whl"
40-
)
41-
_NM_YOLOV5_NIGHTLY = _NM_YOLOV5_TAR_TEMPLATE.format(version="nightly")
42-
43-
44-
def _install_yolov5_and_deps():
45-
46-
import subprocess as _subprocess
47-
import sys as _sys
48-
49-
import sparseml as _sparseml
50-
51-
nm_yolov5_release = (
52-
"nightly" if not _sparseml.is_release else f"v{_sparseml.version_major_minor}"
53-
)
54-
55-
yolov5_requirement = _NM_YOLOV5_TAR_TEMPLATE.format(version=nm_yolov5_release)
56-
57-
try:
58-
_subprocess.check_call(
59-
[
60-
_sys.executable,
61-
"-m",
62-
"pip",
63-
"install",
64-
yolov5_requirement,
65-
]
66-
)
67-
68-
import yolov5 as _yolov5
69-
70-
_LOGGER.info("sparseml-yolov5 and dependencies successfully installed")
71-
except Exception:
72-
raise ValueError(
73-
"Unable to install and import sparseml-yolov5 dependencies check "
74-
"that yolov5 is installed, if not, install via "
75-
f"`pip install {_NM_YOLOV5_NIGHTLY}`"
76-
)
7728

7829

7930
def _check_yolov5_install():
80-
if _yolov5_import_error is not None:
81-
import os
82-
83-
if os.getenv("NM_NO_AUTOINSTALL_YOLOV5", False):
84-
_LOGGER.warning(
85-
"Unable to import, skipping auto installation "
86-
"due to NM_NO_AUTOINSTALL_YOLOV5"
87-
)
88-
# skip any further checks
89-
return
90-
else:
91-
_LOGGER.warning(
92-
"sparseml-yolov5 installation not detected. Installing "
93-
"sparseml-yolov5 dependencies if yolov5 is already "
94-
"installed in the environment, it will be overwritten. Set "
95-
"environment variable NM_NO_AUTOINSTALL_YOLOV5 to disable"
96-
)
97-
_install_yolov5_and_deps()
98-
99-
# re check import after potential install
31+
# check nm-yolov5 is installed
10032
try:
10133
import yolov5 as _yolov5
10234
except Exception:
103-
_LOGGER.warning(
104-
"the neuralmagic fork of yolov5 may not be installed. it can be "
105-
"installed via "
106-
f"`pip install {_NM_YOLOV5_NIGHTLY}`"
35+
raise ImportError(
36+
"Unable to import the neuralmagic fork of yolov5 may not be installed. "
37+
f"it can be installed via `pip install nn-yolov5`"
10738
)
10839

10940

11041
_check_yolov5_install()
42+
43+
from .helpers import *

0 commit comments

Comments
 (0)