diff --git a/eslint-plugin/lib/rules/no-import-all-from-library.js b/eslint-plugin/lib/rules/no-import-all-from-library.js
index 3073e0e..af966c1 100644
--- a/eslint-plugin/lib/rules/no-import-all-from-library.js
+++ b/eslint-plugin/lib/rules/no-import-all-from-library.js
@@ -84,7 +84,13 @@ module.exports = {
       ImportDeclaration(node) {
         const currentLibrary = node.source.value;
 
-        const forbiddenByName = notAllowedLibraries.includes(currentLibrary);
+        const forbiddenByName =
+          notAllowedLibraries.includes(currentLibrary) &&
+          node.specifiers.some(
+            (specifier) =>
+              specifier.type === "ImportDefaultSpecifier" ||
+              specifier.type === "ImportNamespaceSpecifier",
+          );
         const forbiddenByNamespace =
           importByNamespaceNotAllowedLibraries.includes(currentLibrary) &&
           node.specifiers.some(
diff --git a/eslint-plugin/tests/lib/rules/no-import-all-from-library.test.js b/eslint-plugin/tests/lib/rules/no-import-all-from-library.test.js
index e04fc92..f1a44e2 100644
--- a/eslint-plugin/tests/lib/rules/no-import-all-from-library.test.js
+++ b/eslint-plugin/tests/lib/rules/no-import-all-from-library.test.js
@@ -54,6 +54,12 @@ const tests = {
     `
     import map from 'underscore/modules/map.js';
     `,
+    `
+    import { memoize, omitBy, isNil } from 'lodash';
+    `,
+    `
+    import { isEmpty } from 'underscore';
+    `,
   ],
 
   invalid: [