Skip to content

Fix closed school listing issue #396

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
madhurabhujbal merged 5 commits into from
Dec 30, 2025
Merged

Fix closed school listing issue #396

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
2b65d98
Change constants matching header name in schools list
madhurabhujbal Dec 19, 2025
7fec766
Add new school list for only open schools and logs for curropted indexes
madhurabhujbal Dec 29, 2025
5c7752b
Fix the spacing
madhurabhujbal Dec 29, 2025
f247166
Fix the puppy crawl errors
madhurabhujbal Dec 29, 2025
8cb979d
Refine the function for getting boolean for closed school
madhurabhujbal Dec 29, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Binary file modified resources/school_list_2025.tar.gz
View file
Open in desktop
Binary file not shown.
5 changes: 3 additions & 2 deletions src/main/java/uk/ac/cam/cl/dtg/segue/api/Constants.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -549,14 +549,15 @@ public enum EventFilterOption {

// School List loading - raw data
public static final String SCHOOL_URN_FIELDNAME = "urn";
public static final String SCHOOL_ESTABLISHMENT_NAME_FIELDNAME = "school_name";
public static final String SCHOOL_NAME_FIELDNAME = "school_name";
public static final String SCHOOL_POSTCODE_FIELDNAME = "postcode";
public static final String SCHOOL_DATA_SOURCE_FIELDNAME = "data_source";
public static final String SCHOOL_CLOSED_FIELDNAME = "closed";

// School List loading POJO fields
// Note: These are the JSON field names as serialized by Jackson (matches School class field names)
public static final String SCHOOL_URN_FIELDNAME_POJO = "urn";
public static final String SCHOOL_ESTABLISHMENT_NAME_FIELDNAME_POJO = "name";
public static final String SCHOOL_NAME_FIELDNAME_POJO = "name";
public static final String SCHOOL_POSTCODE_FIELDNAME_POJO = "postcode";

// User School Reporting
Expand Down
4 changes: 2 additions & 2 deletions src/main/java/uk/ac/cam/cl/dtg/segue/dao/schools/SchoolListReader.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@

import static uk.ac.cam.cl.dtg.segue.api.Constants.DEFAULT_RESULTS_LIMIT;
import static uk.ac.cam.cl.dtg.segue.api.Constants.SCHOOLS_INDEX_BASE;
import static uk.ac.cam.cl.dtg.segue.api.Constants.SCHOOL_ESTABLISHMENT_NAME_FIELDNAME_POJO;
import static uk.ac.cam.cl.dtg.segue.api.Constants.SCHOOL_NAME_FIELDNAME_POJO;
import static uk.ac.cam.cl.dtg.segue.api.Constants.SCHOOL_POSTCODE_FIELDNAME_POJO;
import static uk.ac.cam.cl.dtg.segue.api.Constants.SCHOOL_URN_FIELDNAME;
import static uk.ac.cam.cl.dtg.segue.api.Constants.SCHOOL_URN_FIELDNAME_POJO;
Expand Down Expand Up @@ -98,7 +98,7 @@ public List<School> findSchoolByNameOrPostCode(final String searchQuery)
List<String> schoolSearchResults = searchProvider.fuzzySearch(
new BasicSearchParameters(SCHOOLS_INDEX_BASE, SchoolsIndexType.SCHOOL_SEARCH.toString(), 0,
DEFAULT_RESULTS_LIMIT),
searchQuery, null, null, SCHOOL_URN_FIELDNAME_POJO, SCHOOL_ESTABLISHMENT_NAME_FIELDNAME_POJO,
searchQuery, null, null, SCHOOL_URN_FIELDNAME_POJO, SCHOOL_NAME_FIELDNAME_POJO,
SCHOOL_POSTCODE_FIELDNAME_POJO
).getResults();

Expand Down
21 changes: 18 additions & 3 deletions src/main/java/uk/ac/cam/cl/dtg/segue/etl/SchoolIndexer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,14 @@ synchronized void indexSchoolsWithSearchProvider() throws UnableToIndexSchoolsEx
}

log.info("Creating schools index with search provider.");
// Clear the existing school index to remove any old/corrupted entries
boolean cleared = es.expungeIndexFromSearchCache(SCHOOLS_INDEX_BASE, SchoolsIndexType.SCHOOL_SEARCH.toString());
if (cleared) {
log.info("Cleared existing school index before reindexing.");
} else {
log.info("School index did not exist or could not be cleared (this is OK for first-time indexing).");
}

List<School> schoolList = this.loadAndBuildSchoolList();
List<Map.Entry<String, String>> indexList = Lists.newArrayList();
ObjectMapper objectMapper = mapperUtils.getSharedContentObjectMapper();
Expand Down Expand Up @@ -138,10 +146,9 @@ private synchronized List<School> loadAndBuildSchoolList() throws UnableToIndexS
.valueOf(schoolArray[fieldNameMapping.get(Constants.SCHOOL_DATA_SOURCE_FIELDNAME)]);

School schoolToSave = new School(schoolArray[fieldNameMapping.get(Constants.SCHOOL_URN_FIELDNAME)],
schoolArray[fieldNameMapping.get(Constants.SCHOOL_ESTABLISHMENT_NAME_FIELDNAME)],
schoolArray[fieldNameMapping.get(Constants.SCHOOL_NAME_FIELDNAME)],
schoolArray[fieldNameMapping.get(Constants.SCHOOL_POSTCODE_FIELDNAME)],
// CSV file contains string "t" and "f" values to denote true and false, but need a boolean:
"t".equals(schoolArray[fieldNameMapping.get(Constants.SCHOOL_CLOSED_FIELDNAME)]),
Boolean.valueOf(schoolArray[fieldNameMapping.get(Constants.SCHOOL_CLOSED_FIELDNAME)]),
source);

if (null == schoolToSave.getPostcode() || schoolToSave.getPostcode().isEmpty()) {
Expand All @@ -153,6 +160,14 @@ private synchronized List<School> loadAndBuildSchoolList() throws UnableToIndexS
// This happens when the school does not have the required data
log.warn("Unable to load the following school into the school list due to missing required fields. {}",
Arrays.toString(schoolArray));
} catch (IllegalArgumentException e) {
// This happens when data_source field is missing, empty, or has an invalid value
log.warn("Unable to load the following school into the school list due to invalid data_source value. "
+ "URN: {}, Error: {}, Row: {}",
schoolArray.length > fieldNameMapping.get(Constants.SCHOOL_URN_FIELDNAME)
? schoolArray[fieldNameMapping.get(Constants.SCHOOL_URN_FIELDNAME)] : "unknown",
e.getMessage(),
Arrays.toString(schoolArray));
}
}
} catch (FileNotFoundException e) {
Expand Down
Loading