Skip to content
This repository was archived by the owner on Jun 3, 2025. It is now read-only.

Commit aee7a0c

Browse files
bfineranbfineran
authored
Revert "remove auto installs and switch to pypi forks for transformers and yolov5 (#1518)" (#1549)
This reverts commit 3688e67.
1 parent 3688e67 commit aee7a0c

File tree

4 files changed

+176
-26
lines changed

4 files changed

+176
-26
lines changed

‎docker/Dockerfile‎

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -95,19 +95,23 @@ ARG MODE
9595
RUN \
9696
if [ -n "$BRANCH" ] ; then \
9797
echo Installing from BRANCH && \
98-
$VENV/bin/pip install --no-cache-dir "./sparseml[onnxruntime,torchvision,transformers,yolov5,ultralytics]"; \
98+
$VENV/bin/pip install --no-cache-dir "./sparseml[onnxruntime,torchvision,ultralytics]"; \
9999
elif [ "$MODE" = "nightly" ] ; then \
100100
if [ -z $VERSION] ; then \
101-
$VENV/bin/pip install --no-cache-dir "sparseml-nightly[onnxruntime,torchvision,transformers,yolov5,ultralytics]"; \
101+
$VENV/bin/pip install --no-cache-dir "sparseml-nightly[onnxruntime,torchvision,ultralytics]"; \
102102
else \
103-
$VENV/bin/pip install --no-cache-dir "sparseml-nightly[onnxruntime,torchvision,transformers,yolov5,ultralytics]==$VERSION"; \
103+
$VENV/bin/pip install --no-cache-dir "sparseml-nightly[onnxruntime,torchvision,ultralytics]==$VERSION"; \
104104
fi; \
105105
elif [ -z $VERSION] ; then \
106-
$VENV/bin/pip install --no-cache-dir "sparseml[onnxruntime,torchvision,transformers,yolov5,ultralytics]"; \
106+
$VENV/bin/pip install --no-cache-dir "sparseml[onnxruntime,torchvision,ultralytics]"; \
107107
else \
108-
$VENV/bin/pip install --no-cache-dir "sparseml[onnxruntime,torchvision,transformers,yolov5,ultralytics]==$VERSION"; \
108+
$VENV/bin/pip install --no-cache-dir "sparseml[onnxruntime,torchvision,ultralytics]==$VERSION"; \
109109
fi;
110110

111+
RUN sparseml.transformers.question_answering --help \
112+
&& sparseml.yolov5.train --help \
113+
&& sparseml.ultralytics.train --help
114+
111115

112116
FROM cuda_builder AS container_branch_dev
113117
ARG VENV

‎setup.py‎

Lines changed: 0 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -71,16 +71,6 @@
7171
"torchvision>=0.3.0,<=0.14",
7272
]
7373
_pytorch_vision_deps = _pytorch_deps + ["torchvision>=0.3.0,<=0.14"]
74-
_transformers_deps = _pytorch_deps + [
75-
f"{'nm-transformers' if is_release else 'nm-transformers-nightly'}"
76-
f"~={version_nm_deps}",
77-
"datasets<=1.18.4",
78-
"scikit-learn",
79-
"seqeval",
80-
]
81-
_yolov5_deps = _pytorch_vision_deps + [
82-
f"{'nm-yolov5' if is_release else 'nm-yolov5-nightly'}~={version_nm_deps}"
83-
]
8474
_tensorflow_v1_deps = ["tensorflow<2.0.0", "tensorboard<2.0.0", "tf2onnx>=1.0.0,<1.6"]
8575
_tensorflow_v1_gpu_deps = [
8676
"tensorflow-gpu<2.0.0",
@@ -142,12 +132,10 @@ def _setup_extras() -> Dict:
142132
"torch": _pytorch_deps,
143133
"torch_all": _pytorch_all_deps,
144134
"torchvision": _pytorch_vision_deps,
145-
"transformers": _transformers_deps,
146135
"tf_v1": _tensorflow_v1_deps,
147136
"tf_v1_gpu": _tensorflow_v1_gpu_deps,
148137
"tf_keras": _keras_deps,
149138
"ultralytics": _ultralytics_deps,
150-
"yolov5": _yolov5_deps,
151139
}
152140

153141

‎src/sparseml/transformers/__init__.py‎

Lines changed: 94 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -20,20 +20,111 @@
2020

2121
import logging as _logging
2222

23+
import pkg_resources
2324
from sparseml.analytics import sparseml_analytics as _analytics
2425

2526

2627
_analytics.send_event("python__transformers__init")
2728

29+
_EXPECTED_VERSION = "4.23.1"
30+
2831

2932
_LOGGER = _logging.getLogger(__name__)
33+
_NM_TRANSFORMERS_TAR_TEMPLATE = (
34+
"https://github.com/neuralmagic/transformers/releases/download/"
35+
"{version}/transformers-4.23.1-py3-none-any.whl"
36+
)
37+
_NM_TRANSFORMERS_NIGHTLY = _NM_TRANSFORMERS_TAR_TEMPLATE.format(version="nightly")
38+
39+
40+
def _install_transformers_and_deps():
41+
42+
import subprocess as _subprocess
43+
import sys as _sys
44+
45+
import sparseml as _sparseml
46+
47+
nm_transformers_release = (
48+
"nightly" if not _sparseml.is_release else f"v{_sparseml.version_major_minor}"
49+
)
50+
transformers_requirement = _NM_TRANSFORMERS_TAR_TEMPLATE.format(
51+
version=nm_transformers_release
52+
)
53+
try:
54+
_subprocess.check_call(
55+
[
56+
_sys.executable,
57+
"-m",
58+
"pip",
59+
"install",
60+
transformers_requirement,
61+
"datasets<=1.18.4",
62+
"scikit-learn",
63+
"seqeval",
64+
]
65+
)
66+
67+
import transformers as _transformers
68+
69+
_LOGGER.info("sparseml-transformers and dependencies successfully installed")
70+
except Exception:
71+
raise ValueError(
72+
"Unable to install and import sparseml-transformers dependencies check "
73+
"that transformers is installed, if not, install via "
74+
f"`pip install {_NM_TRANSFORMERS_NIGHTLY}`"
75+
)
3076

3177

3278
def _check_transformers_install():
33-
# check for NM integration in transformers version
34-
import transformers as _transformers
79+
transformers_version = next(
80+
(
81+
pkg.version
82+
for pkg in pkg_resources.working_set
83+
if pkg.project_name.lower() == "transformers"
84+
),
85+
None,
86+
)
87+
88+
# Either no transformers install is found or wrong version installed
89+
if transformers_version != _EXPECTED_VERSION:
90+
import os
91+
92+
if os.getenv("NM_NO_AUTOINSTALL_TRANSFORMERS", False):
93+
_LOGGER.warning(
94+
"Unable to import, skipping auto installation "
95+
"due to NM_NO_AUTOINSTALL_TRANSFORMERS"
96+
)
97+
# skip any further checks
98+
return
99+
else:
100+
_LOGGER.warning(
101+
f"sparseml-transformers v{_EXPECTED_VERSION} installation not "
102+
f"detected. Installing sparseml-transformers v{_EXPECTED_VERSION} "
103+
"dependencies if transformers is already installed in the "
104+
"environment, it will be overwritten. Set environment variable "
105+
"NM_NO_AUTOINSTALL_TRANSFORMERS to disable"
106+
)
107+
_install_transformers_and_deps()
108+
109+
else:
110+
import transformers as _transformers
111+
112+
# Edge case where user has expected version of transformers installed, but
113+
# not the nm integrated one
114+
if not _transformers.NM_INTEGRATED:
115+
_install_transformers_and_deps()
116+
raise RuntimeError(
117+
"Installed transformers package has been overwritten with "
118+
"sparseml-transformers. Stopping process as this is likely to cause "
119+
"import issues. Please re-run command"
120+
)
121+
122+
# re check import after potential install
123+
try:
124+
import transformers as _transformers
35125

36-
if not _transformers.NM_INTEGRATED:
126+
assert _transformers.NM_INTEGRATED
127+
except Exception:
37128
_LOGGER.warning(
38129
"the neuralmagic fork of transformers may not be installed. it can be "
39130
"installed via "

‎src/sparseml/yolov5/__init__.py‎

Lines changed: 73 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -21,23 +21,90 @@
2121

2222
from sparseml.analytics import sparseml_analytics as _analytics
2323

24+
from .helpers import *
25+
2426

2527
_analytics.send_event("python__yolov5__init")
2628

29+
try:
30+
import yolov5 as _yolov5
31+
32+
_yolov5_import_error = None
33+
except Exception as _yolov5_import_err:
34+
_yolov5_import_error = _yolov5_import_err
35+
2736
_LOGGER = _logging.getLogger(__name__)
37+
_NM_YOLOV5_TAR_TEMPLATE = (
38+
"https://github.com/neuralmagic/yolov5/releases/download/"
39+
"{version}/yolov5-6.2.0-py3-none-any.whl"
40+
)
41+
_NM_YOLOV5_NIGHTLY = _NM_YOLOV5_TAR_TEMPLATE.format(version="nightly")
42+
43+
44+
def _install_yolov5_and_deps():
45+
46+
import subprocess as _subprocess
47+
import sys as _sys
48+
49+
import sparseml as _sparseml
50+
51+
nm_yolov5_release = (
52+
"nightly" if not _sparseml.is_release else f"v{_sparseml.version_major_minor}"
53+
)
54+
55+
yolov5_requirement = _NM_YOLOV5_TAR_TEMPLATE.format(version=nm_yolov5_release)
56+
57+
try:
58+
_subprocess.check_call(
59+
[
60+
_sys.executable,
61+
"-m",
62+
"pip",
63+
"install",
64+
yolov5_requirement,
65+
]
66+
)
67+
68+
import yolov5 as _yolov5
69+
70+
_LOGGER.info("sparseml-yolov5 and dependencies successfully installed")
71+
except Exception:
72+
raise ValueError(
73+
"Unable to install and import sparseml-yolov5 dependencies check "
74+
"that yolov5 is installed, if not, install via "
75+
f"`pip install {_NM_YOLOV5_NIGHTLY}`"
76+
)
2877

2978

3079
def _check_yolov5_install():
31-
# check nm-yolov5 is installed
80+
if _yolov5_import_error is not None:
81+
import os
82+
83+
if os.getenv("NM_NO_AUTOINSTALL_YOLOV5", False):
84+
_LOGGER.warning(
85+
"Unable to import, skipping auto installation "
86+
"due to NM_NO_AUTOINSTALL_YOLOV5"
87+
)
88+
# skip any further checks
89+
return
90+
else:
91+
_LOGGER.warning(
92+
"sparseml-yolov5 installation not detected. Installing "
93+
"sparseml-yolov5 dependencies if yolov5 is already "
94+
"installed in the environment, it will be overwritten. Set "
95+
"environment variable NM_NO_AUTOINSTALL_YOLOV5 to disable"
96+
)
97+
_install_yolov5_and_deps()
98+
99+
# re check import after potential install
32100
try:
33101
import yolov5 as _yolov5
34102
except Exception:
35-
raise ImportError(
36-
"Unable to import the neuralmagic fork of yolov5 may not be installed. "
37-
f"it can be installed via `pip install nn-yolov5`"
103+
_LOGGER.warning(
104+
"the neuralmagic fork of yolov5 may not be installed. it can be "
105+
"installed via "
106+
f"`pip install {_NM_YOLOV5_NIGHTLY}`"
38107
)
39108

40109

41110
_check_yolov5_install()
42-
43-
from .helpers import *

0 commit comments

Comments
 (0)