diff --git a/README.rst b/README.rst index 4ad75c2b6..1dd030acd 100644 --- a/README.rst +++ b/README.rst @@ -71,8 +71,8 @@ or check out summaries of the validation results in the `validation report `_: +If you would like to install and run COMPASS from source, we recommend using +`pixi `_: .. code-block:: bash git clone git@github.com:NatLabRockies/COMPASS.git; cd COMPASS pixi run compass -For detailed instructions and troubleshooting, see the `installation documentation `_. +For detailed instructions and troubleshooting, see the +`installation documentation `_. Quickstart diff --git a/compass/plugin/ordinance.py b/compass/plugin/ordinance.py index 08ff7b017..16c00c6ad 100644 --- a/compass/plugin/ordinance.py +++ b/compass/plugin/ordinance.py @@ -835,9 +835,11 @@ async def parse_single_doc_for_structured_data(self, extraction_context): doc_for_extraction, out_fn_stem=self.jurisdiction.full_name ) extraction_context.attrs["structured_data"] = data_df + n_feats = num_ordinances_dataframe(data_df) + extraction_context.attrs["num_features_extracted"] = n_feats logger.info( "%d ordinance value(s) found for %s from doc:\n%s. ", - num_ordinances_dataframe(data_df), + n_feats, self.jurisdiction.full_name, doc_for_extraction, ) @@ -897,9 +899,11 @@ async def parse_multi_doc_context_for_structured_data( ) extraction_context.attrs["structured_data"] = data_df + n_feats = num_ordinances_dataframe(data_df) + extraction_context.attrs["num_features_extracted"] = n_feats logger.info( "%d ordinance value(s) found for %s in %d docs. ", - num_ordinances_dataframe(data_df), + n_feats, self.jurisdiction.full_name, extraction_context.num_documents, ) @@ -938,6 +942,7 @@ async def parse_multi_doc_concat(self, extraction_context): data_dfs = await asyncio.gather(*tasks) all_data = [] + total_features = 0 for doc_ind, (data_df, doc) in enumerate( zip(data_dfs, extraction_context, strict=True), start=1 ): @@ -950,9 +955,11 @@ async def parse_multi_doc_concat(self, extraction_context): await extraction_context.mark_doc_as_data_source( doc, out_fn_stem=f"{self.jurisdiction.full_name}_{doc_ind}" ) + num_found = num_ordinances_dataframe(data_df) + total_features += num_found logger.info( "%d ordinance value(s) found for %s from doc:\n%s. ", - num_ordinances_dataframe(data_df), + num_found, self.jurisdiction.full_name, doc, ) @@ -968,6 +975,7 @@ async def parse_multi_doc_concat(self, extraction_context): extraction_context.attrs["structured_data"] = pd.concat( all_data, ignore_index=True ) + extraction_context.attrs["num_features_extracted"] = total_features return extraction_context async def parse_multi_doc_merge(self, extraction_context): diff --git a/compass/services/openai.py b/compass/services/openai.py index a089d6cc9..2205c4451 100644 --- a/compass/services/openai.py +++ b/compass/services/openai.py @@ -218,6 +218,7 @@ def _update_pb_cost(self, response): openai.RateLimitError, openai.APITimeoutError, openai.BadRequestError, + openai.APIConnectionError, ), ) async def _call_gpt(self, **kwargs): diff --git a/compass/services/threaded.py b/compass/services/threaded.py index 4fe6c2d58..f10a15589 100644 --- a/compass/services/threaded.py +++ b/compass/services/threaded.py @@ -645,20 +645,28 @@ def _compile_doc_info(doc): out_fp = doc.attrs.get("source_fp", doc.attrs.get("out_fp")) return { "source": doc.attrs.get("source"), + "num_features_extracted": doc.attrs.get("num_features_extracted"), "effective_year": year if year is not None and year > 0 else None, "effective_month": month if month is not None and month > 0 else None, "effective_day": day if day is not None and day > 0 else None, "ord_filename": Path(out_fp or "unknown").name, + "doc_type": doc.attrs.get("doc_type"), "num_pages": doc.attrs.get("num_pages", len(doc.pages)), "checksum": doc.attrs.get("checksum"), "is_pdf": is_pdf_doc(doc), "from_ocr": doc.attrs.get("from_ocr", False), + "conversion_time_seconds": doc.attrs.get("conversion_time_seconds"), + "conversion_status": doc.attrs.get("conversion_status"), + "mean_confidence": doc.attrs.get("mean_confidence"), + "low_score_confidence": doc.attrs.get("low_score_confidence"), + "collection_step_rank": doc.attrs.get("collection_step_rank"), "relevant_text_ngram_score": doc.attrs.get( "relevant_text_ngram_score" ), "permitted_use_text_ngram_score": doc.attrs.get( "permitted_use_text_ngram_score" ), + "from_steps": doc.attrs.get("from_steps"), }