SED-4488 Poor performance of the steps table when filtering by statuses

step-framework-collections-mongodb/src/main/java/step/core/collections/mongodb/MongoDBFilterFactory.java

@@ -83,7 +83,16 @@ public Bson buildFilter(Filter filter) {
 		} else if (filter instanceof Exists) {
 			Exists existsFilter = (Exists) filter;
 			return com.mongodb.client.model.Filters.exists(existsFilter.getField());
-		} else {
+		} else if (filter instanceof In) {
+			In inFilter = (In) filter;
+			String field = inFilter.getField();
+			List<Object> values = inFilter.getValues();
+			if(field.equals("id")) {
+				field = "_id";
+				values = values.stream().map(s -> (s instanceof String) ? new ObjectId((String) s) : s).collect(Collectors.toList());
+			}
+			return com.mongodb.client.model.Filters.in(field, values);
+		}else {
 			throw new IllegalArgumentException("Unsupported filter type " + filter.getClass());
 		}
 	}

step-framework-collections-postgresql/src/main/java/step/core/collections/postgresql/PostgreSQLCollection.java

@@ -106,8 +106,8 @@ public long count(Filter filter, Integer limit) {
 		String query = "SELECT count(d.*) FROM (SELECT id FROM " + collectionNameStr +
 				" WHERE " + filterToWhereClause(filter) + " LIMIT " + limit + ") d";
 		try (Connection connection = ds.getConnection();
-			 Statement statement = connection.createStatement()) {
-			ResultSet resultSet = statement.executeQuery(query);
+			 Statement statement = connection.createStatement();
+			 ResultSet resultSet = statement.executeQuery(query)) {
 			if (resultSet.next()) {
 				return resultSet.getInt(1);
 			} else {

step-framework-collections-postgresql/src/main/java/step/core/collections/postgresql/PostgreSQLFilterFactory.java

@@ -91,6 +91,13 @@ public String buildFilter(Filter filter) {
 		} else if (filter instanceof Exists) {
 			Exists existsFilter = (Exists) filter;
 			return formatField(existsFilter.getField(),false) + " IS NOT NULL ";
+		} else if (filter instanceof In) {
+			In inFilter = (In) filter;
+			//In filter values implementation only support comparison as Strings, so the field and values are formated to string
+			String values = inFilter.getValues().stream()
+					.map(this::formatInValue) // Escape single quotes for SQL
+					.collect(Collectors.joining(",", "(", ")"));
+			return formatField(inFilter.getField(), true) + " IN " + values + " ";
 		} else {
 			throw new IllegalArgumentException("Unsupported filter type " + filter.getClass());
 		}
@@ -106,6 +113,18 @@ private String notPsqlClause(Not notFilter, List<String> childerPojoFilters) {
 		}
 	}
 
+	private String formatInValue(Object expectedValue) {
+		String result;
+		if (expectedValue instanceof String) {
+			result =  escapeValue((String) expectedValue);
+		} else if (expectedValue instanceof ObjectId) {
+			result = ((ObjectId) expectedValue).toHexString();
+		} else {
+			result = expectedValue.toString();
+		}
+		return "'" + result + "'";
+	}
+
 	private String escapeValue(String expectedValue) {
 		return expectedValue.replaceAll("'","''");
 	}

step-framework-collections/src/main/java/step/core/accessors/AbstractAccessor.java

@@ -76,9 +76,8 @@ public T findByCriteria(Map<String, String> criteria) {
 
 	@Override
 	public Stream<T> findByIds(List<String> ids) {
-		List<String> idsList = ids.stream()
+		List<Object> idsList = ids.stream()
 			.map(ObjectId::new)
-			.map(ObjectId::toString)
 			.collect(Collectors.toList());
 		return collectionDriver.find(Filters.in("id", idsList), null, null, null, 0);
 	}

step-framework-collections/src/main/java/step/core/collections/PojoFilters.java

@@ -26,6 +26,7 @@
 import java.lang.reflect.InvocationTargetException;
 import java.math.BigDecimal;
 import java.util.List;
+import java.util.Objects;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
@@ -76,6 +77,8 @@ public PojoFilter<POJO> buildFilter(Filter filter) {
 				return new GtePojoFilter<>((Gte) filter);
 			} else if (filter instanceof Exists) {
 				return new ExistsPojoFilter<>((Exists) filter);
+			} else if (filter instanceof In) {
+				return new InPojoFilter<>((In) filter);
 			} else {
 				throw new IllegalArgumentException("Unsupported filter type " + filter.getClass());
 			}
@@ -173,27 +176,31 @@ public boolean test(T t) {
 			try {
 				String field = equalsFilter.getField();
 				Object beanProperty = getBeanProperty(t, field);
-				if(expectedValue != null) {
-					if(expectedValue instanceof Number) {
-						if(beanProperty != null) {
-							return new BigDecimal(expectedValue.toString()).compareTo(new BigDecimal(beanProperty.toString()))==0;
-						} else {
-							return false;
-						}
-					} if (expectedValue instanceof String && beanProperty!= null && beanProperty.getClass().isEnum())  {
-						return expectedValue.equals(beanProperty.toString());
-					} else {
-						return expectedValue.equals(beanProperty);
-					}
-				} else {
-					return beanProperty == null; 
-				}
+				return testProperty(beanProperty);
 			} catch (NoSuchMethodException e) {
 				return (expectedValue == null);
 			} catch (IllegalAccessException | InvocationTargetException e) {
 				return false;
 			}
 		}
+
+		public boolean testProperty(Object beanProperty) {
+			if(expectedValue != null) {
+				if(expectedValue instanceof Number) {
+					if(beanProperty instanceof Number) {
+						return new BigDecimal(expectedValue.toString()).compareTo(new BigDecimal(beanProperty.toString()))==0;
+					} else {
+						return false;
+					}
+				} if (expectedValue instanceof String && beanProperty!= null && beanProperty.getClass().isEnum())  {
+					return expectedValue.equals(beanProperty.toString());
+				} else {
+					return expectedValue.equals(beanProperty);
+				}
+			} else {
+				return beanProperty == null;
+			}
+		}
 	}
 
 	public static class RegexPojoFilter<T> implements PojoFilter<T> {
@@ -356,6 +363,38 @@ public boolean test(T t) {
 			return beanProperty != null;
 		}
 	}
+
+	public static class InPojoFilter<T> implements PojoFilter<T> {
+
+		private final String field;
+		private final List<EqualsPojoFilter<T>> equalFilters;
+
+		public InPojoFilter(In inFilter) {
+			super();
+			field = inFilter.getField();
+			equalFilters = inFilter.getValues().stream().map(v -> new EqualsPojoFilter<T>(new Equals(field, v))).collect(Collectors.toList());
+		}
+
+		@Override
+		public boolean test(T t) {
+			try {
+				Object beanProperty = getBeanProperty(t, field);
+				return equalFilters.stream().anyMatch(eq -> eq.testProperty(beanProperty));
+			} catch (IllegalAccessException | InvocationTargetException | NoSuchMethodException e) {
+				return false;
+			}
+		}
+
+		public boolean compareNonNullValues(Object beanValue, Object inValue) {
+			if (beanValue instanceof ObjectId && inValue instanceof String) {
+				return ((ObjectId) beanValue).toHexString().equals(inValue);
+			} else if (beanValue instanceof String && inValue instanceof ObjectId) {
+				return ((ObjectId) inValue).toHexString().equals(beanValue);
+			} else {
+				return beanValue.equals(inValue);
+			}
+		}
+	}
 
 	private static Object getBeanProperty(Object t, String fieldName)
 			throws IllegalAccessException, InvocationTargetException, NoSuchMethodException {

step-framework-collections/src/main/java/step/core/ql/OQLFilterVisitor.java

@@ -106,7 +106,7 @@ public Filter visitInExpr(InExprContext ctx) {
     }
 
     protected Filter processInExpr(InExprContext ctx, String text0) {
-        List<String> ins = ctx.STRING().stream().map(tn -> unescapeStringIfNecessary(tn.getText())).collect(Collectors.toList());
+        List<Object> ins = ctx.STRING().stream().map(tn -> unescapeStringIfNecessary(tn.getText())).collect(Collectors.toList());
         return Filters.in(text0, ins);
     }
 

step-framework-collections/src/test/java/step/core/collections/AbstractCollectionTest.java

@@ -268,7 +268,7 @@ public void testFindFilters() {
 		Bean bean2 = new Bean();
 		bean2.setProperty1("My property 2");
 		bean2.setLongProperty(21l);
-		bean2.setBooleanProperty(false);
+		bean2.setBooleanProperty(true);
 		bean2.addAttribute("MyAtt1", "My value 2");
 		bean2.addAttribute("MyAtt2", "My other value");
 		collection.save(bean2);
@@ -291,16 +291,21 @@ public void testFindFilters() {
 
 		// Equals boolean
 		result = collection.find(Filters.equals("booleanProperty", false), new SearchOrder("MyAtt1", 1), null, null, 0).collect(Collectors.toList());
+		assertEquals(1, result.size());
 		assertEquals(bean.getId(), result.get(0).getId());
 
 		// Equals long
 		result = collection.find(Filters.equals("longProperty", 11), new SearchOrder("MyAtt1", 1), null, null, 0).collect(Collectors.toList());
 		assertEquals(bean.getId(), result.get(0).getId());
 
+		result = collection.find(Filters.equals("longProperty", 11L), new SearchOrder("MyAtt1", 1), null, null, 0).collect(Collectors.toList());
+		assertEquals(bean.getId(), result.get(0).getId());
+
 		//is null
 		result = collection.find(Filters.equals("missingField", (String) null), new SearchOrder("MyAtt1", 1), null, null, 0).collect(Collectors.toList());
 		assertEquals(bean.getId(), result.get(0).getId());
 
+		//and
 		result = collection.find(Filters.and(List.of(Filters.gte("longProperty", 11),Filters.lt("longProperty",21))), new SearchOrder("MyAtt1", 1), null, null, 0).collect(Collectors.toList());
 		assertEquals(bean.getId(), result.get(0).getId());
 		assertEquals(1, result.size());
@@ -309,6 +314,42 @@ public void testFindFilters() {
 		assertEquals(bean2.getId(), result.get(0).getId());
 		assertEquals(1, result.size());
 
+		//or
+		result = collection.find(Filters.or(List.of(Filters.equals("property1", "My property 1"),Filters.equals("property1", "My property 2"))), new SearchOrder("MyAtt1", 1), null, null, 0).collect(Collectors.toList());
+		assertEquals(2, result.size());
+
+		//in
+		result = collection.find(Filters.in("property1", List.of("My property 1")), new SearchOrder("MyAtt1", 1), null, null, 0).collect(Collectors.toList());
+		assertEquals(1, result.size());
+		assertEquals(bean.getId(), result.get(0).getId());
+
+		result = collection.find(Filters.in("property1", List.of("My property 1", "My property 2")), new SearchOrder("MyAtt1", 1), null, null, 0).collect(Collectors.toList());
+		assertEquals(2, result.size());
+		assertEquals(bean.getId(), result.get(0).getId());
+
+		//in boolean + mix in values
+		result = collection.find(Filters.in("booleanProperty", List.of("My property 1", true, 12)), new SearchOrder("MyAtt1", 1), null, null, 0).collect(Collectors.toList());
+		assertEquals(1, result.size());
+		assertEquals(bean2.getId(), result.get(0).getId());
+
+		//in long + mix in values
+		result = collection.find(Filters.in("longProperty", List.of("My property 1", true, 21)), new SearchOrder("MyAtt1", 1), null, null, 0).collect(Collectors.toList());
+		assertEquals(1, result.size());
+		assertEquals(bean2.getId(), result.get(0).getId());
+
+		result = collection.find(Filters.in("property1", List.of("My property 1")), new SearchOrder("MyAtt1", 1), null, null, 0).collect(Collectors.toList());
+		assertEquals(1, result.size());
+		assertEquals(bean.getId(), result.get(0).getId());
+
+		//in IDs
+		result = collection.find(Filters.in("id", List.of(bean.getId(), bean2.getId())), new SearchOrder("MyAtt1", 1), null, null, 0).collect(Collectors.toList());
+		assertEquals(2, result.size());
+		assertEquals(bean.getId(), result.get(0).getId());
+
+		result = collection.find(Filters.in("id", List.of(bean.getId().toHexString())), new SearchOrder("MyAtt1", 1), null, null, 0).collect(Collectors.toList());
+		assertEquals(1, result.size());
+		assertEquals(bean.getId(), result.get(0).getId());
+
 		//Not equal
 		result = collection.find(Filters.not(Filters.equals("attributes.MyAtt2", "My value 1")), new SearchOrder("MyAtt1", 1), null, null, 0).collect(Collectors.toList());
 		assertEquals(2, result.size());
@@ -328,6 +369,8 @@ public void testFindFilters() {
 		assertThrows(Throwable.class, () -> collection.find(Filters.and(new ArrayList<>()), null, null, null, 0).collect(Collectors.toList()));
 
 		assertThrows(Throwable.class, () -> collection.find(Filters.or(new ArrayList<>()), null, null, null, 0).collect(Collectors.toList()));
+
+		assertThrows(Throwable.class, () -> collection.find(Filters.in("test", new ArrayList<>()), null, null, null, 0).collect(Collectors.toList()));
 	}
 
 	@Test

step-framework-collections/src/test/java/step/core/ql/OQLAttributesAwareFilterVisitorTest.java

@@ -5,7 +5,7 @@
 import org.junit.Test;
 import step.core.collections.Filter;
 import step.core.collections.filters.Equals;
-import step.core.collections.filters.Or;
+import step.core.collections.filters.In;
 import step.core.collections.filters.True;
 
 import java.util.Arrays;
@@ -66,11 +66,12 @@ public void transformWithInCriteriaTest() {
         OQLParser.ParseContext context = parse("field1 in ( \"prop1\", \"prop2\" )");
         OQLAttributesAwareFilterVisitor visitor = new OQLAttributesAwareFilterVisitor((key) -> "prefix." + key);
         Filter filter = visitor.visit(context.getChild(0));
-        Assert.assertTrue(filter instanceof Or);
-        Or orFilter = (Or) filter;
-        Assert.assertEquals(2, orFilter.getChildren().size());
-        Equals prop1Equals = (Equals) orFilter.getChildren().get(0);
-        Assert.assertEquals("prefix.field1", prop1Equals.getField());
+        Assert.assertTrue(filter instanceof In);
+        In inFilter = (In) filter;
+        Assert.assertEquals(2, inFilter.getValues().size());
+        Assert.assertEquals("prefix.field1", inFilter.getField());
+        Assert.assertEquals("prop1", inFilter.getValues().get(0));
+        Assert.assertEquals("prop2", inFilter.getValues().get(1));
     }
 
     private static OQLParser.ParseContext parse(String expression) {

step-framework-model/src/main/java/step/core/collections/Filter.java

@@ -40,7 +40,8 @@
         @JsonSubTypes.Type(Or.class),
         @JsonSubTypes.Type(Regex.class),
         @JsonSubTypes.Type(True.class),
-        @JsonSubTypes.Type(Exists.class)
+        @JsonSubTypes.Type(Exists.class),
+        @JsonSubTypes.Type(In.class)
 })
 public interface Filter {
 

step-framework-model/src/main/java/step/core/collections/Filters.java

@@ -95,14 +95,6 @@ public static Gte gte(String field, long value) {
 		return new Gte(field, value);
 	}
 
-	public static Filter in(String field, List<String> values) {
-		List<Filter> filters = new ArrayList<>();
-		for (String v : values) {
-			filters.add(Filters.equals(field, v));
-		}
-		return (filters.isEmpty()) ? Filters.falseFilter() : Filters.or(filters);
-	}
-
 	public static Regex regex(String field, String expression, boolean caseSensitive) {
 		return new Regex(field, expression, caseSensitive);
 	}
@@ -115,6 +107,10 @@ public static Exists exists(String field) {
 		return new Exists(field);
 	}
 
+	public static In in(String field, List<Object> values) {
+		return new In(field, values);
+	}
+
 	/**
 	 * Helper method to extract all attributes (fields) on which the filters are applied
 	 * @param filter the filter for which with extract the list of attributes

step-framework-model/src/main/java/step/core/collections/filters/In.java

@@ -0,0 +1,55 @@
+package step.core.collections.filters;
+
+import java.util.List;
+import java.util.Objects;
+
+public class In extends AbstractAtomicFilter {
+
+    private String field;
+    private List<Object> values;
+
+    public In() {
+        super();
+    }
+
+    public In(String field, List<Object> values) {
+        super();
+        Objects.requireNonNull(field, "The field of the In filter cannot be null.");
+        Objects.requireNonNull(values, "The list of values of the In filter cannot be null.");
+        if (values.isEmpty()) {
+            throw new IllegalArgumentException("The list of values of the In filter cannot be empty.");
+        }
+        this.field = field;
+        this.values = values;
+    }
+
+    @Override
+    public String getField() {
+        return field;
+    }
+
+    public List<Object> getValues() {
+        return values;
+    }
+
+    public void setField(String field) {
+        this.field = field;
+    }
+
+    public void setValues(List<Object> values) {
+        this.values = values;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        In in = (In) o;
+        return values == in.values && Objects.equals(field, in.field);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(field, values);
+    }
+}