diff --git a/README.rst b/README.rst
index 4ad75c2b6..1dd030acd 100644
--- a/README.rst
+++ b/README.rst
@@ -71,8 +71,8 @@ or check out summaries of the validation results in the `validation report `_:
+If you would like to install and run COMPASS from source, we recommend using
+`pixi `_:
.. code-block:: bash
git clone git@github.com:NatLabRockies/COMPASS.git; cd COMPASS
pixi run compass
-For detailed instructions and troubleshooting, see the `installation documentation `_.
+For detailed instructions and troubleshooting, see the
+`installation documentation `_.
Quickstart
diff --git a/compass/plugin/ordinance.py b/compass/plugin/ordinance.py
index 08ff7b017..16c00c6ad 100644
--- a/compass/plugin/ordinance.py
+++ b/compass/plugin/ordinance.py
@@ -835,9 +835,11 @@ async def parse_single_doc_for_structured_data(self, extraction_context):
doc_for_extraction, out_fn_stem=self.jurisdiction.full_name
)
extraction_context.attrs["structured_data"] = data_df
+ n_feats = num_ordinances_dataframe(data_df)
+ extraction_context.attrs["num_features_extracted"] = n_feats
logger.info(
"%d ordinance value(s) found for %s from doc:\n%s. ",
- num_ordinances_dataframe(data_df),
+ n_feats,
self.jurisdiction.full_name,
doc_for_extraction,
)
@@ -897,9 +899,11 @@ async def parse_multi_doc_context_for_structured_data(
)
extraction_context.attrs["structured_data"] = data_df
+ n_feats = num_ordinances_dataframe(data_df)
+ extraction_context.attrs["num_features_extracted"] = n_feats
logger.info(
"%d ordinance value(s) found for %s in %d docs. ",
- num_ordinances_dataframe(data_df),
+ n_feats,
self.jurisdiction.full_name,
extraction_context.num_documents,
)
@@ -938,6 +942,7 @@ async def parse_multi_doc_concat(self, extraction_context):
data_dfs = await asyncio.gather(*tasks)
all_data = []
+ total_features = 0
for doc_ind, (data_df, doc) in enumerate(
zip(data_dfs, extraction_context, strict=True), start=1
):
@@ -950,9 +955,11 @@ async def parse_multi_doc_concat(self, extraction_context):
await extraction_context.mark_doc_as_data_source(
doc, out_fn_stem=f"{self.jurisdiction.full_name}_{doc_ind}"
)
+ num_found = num_ordinances_dataframe(data_df)
+ total_features += num_found
logger.info(
"%d ordinance value(s) found for %s from doc:\n%s. ",
- num_ordinances_dataframe(data_df),
+ num_found,
self.jurisdiction.full_name,
doc,
)
@@ -968,6 +975,7 @@ async def parse_multi_doc_concat(self, extraction_context):
extraction_context.attrs["structured_data"] = pd.concat(
all_data, ignore_index=True
)
+ extraction_context.attrs["num_features_extracted"] = total_features
return extraction_context
async def parse_multi_doc_merge(self, extraction_context):
diff --git a/compass/services/openai.py b/compass/services/openai.py
index a089d6cc9..2205c4451 100644
--- a/compass/services/openai.py
+++ b/compass/services/openai.py
@@ -218,6 +218,7 @@ def _update_pb_cost(self, response):
openai.RateLimitError,
openai.APITimeoutError,
openai.BadRequestError,
+ openai.APIConnectionError,
),
)
async def _call_gpt(self, **kwargs):
diff --git a/compass/services/threaded.py b/compass/services/threaded.py
index 4fe6c2d58..f10a15589 100644
--- a/compass/services/threaded.py
+++ b/compass/services/threaded.py
@@ -645,20 +645,28 @@ def _compile_doc_info(doc):
out_fp = doc.attrs.get("source_fp", doc.attrs.get("out_fp"))
return {
"source": doc.attrs.get("source"),
+ "num_features_extracted": doc.attrs.get("num_features_extracted"),
"effective_year": year if year is not None and year > 0 else None,
"effective_month": month if month is not None and month > 0 else None,
"effective_day": day if day is not None and day > 0 else None,
"ord_filename": Path(out_fp or "unknown").name,
+ "doc_type": doc.attrs.get("doc_type"),
"num_pages": doc.attrs.get("num_pages", len(doc.pages)),
"checksum": doc.attrs.get("checksum"),
"is_pdf": is_pdf_doc(doc),
"from_ocr": doc.attrs.get("from_ocr", False),
+ "conversion_time_seconds": doc.attrs.get("conversion_time_seconds"),
+ "conversion_status": doc.attrs.get("conversion_status"),
+ "mean_confidence": doc.attrs.get("mean_confidence"),
+ "low_score_confidence": doc.attrs.get("low_score_confidence"),
+ "collection_step_rank": doc.attrs.get("collection_step_rank"),
"relevant_text_ngram_score": doc.attrs.get(
"relevant_text_ngram_score"
),
"permitted_use_text_ngram_score": doc.attrs.get(
"permitted_use_text_ngram_score"
),
+ "from_steps": doc.attrs.get("from_steps"),
}