Minor extraction updates

README.rst

@@ -71,8 +71,8 @@ or check out summaries of the validation results in the `validation report <http
 Where is the extracted ordinance data?
 ======================================
 The National Laboratories of the Rockies (NLR) typically runs the COMPASS pipeline
-annually and publishes refreshed datasets to OpenEI. The latest published ordinance datasets
-are available here:
+annually and publishes refreshed datasets to OpenEI. The latest published ordinance
+datasets are available here:
 
 - Solar: https://data.openei.org/submissions/8602
 - Wind: https://data.openei.org/submissions/8519
@@ -86,14 +86,16 @@ The quickest way to install COMPASS for users is from PyPI:
 
     pip install infra-compass
 
-If you would like to install and run COMPASS from source, we recommend using `pixi <https://pixi.sh/latest/>`_:
+If you would like to install and run COMPASS from source, we recommend using
+`pixi <https://pixi.sh/latest/>`_:
 
 .. code-block:: bash
 
     git clone git@github.com:NatLabRockies/COMPASS.git; cd COMPASS
     pixi run compass
 
-For detailed instructions and troubleshooting, see the `installation documentation <https://natlabrockies.github.io/COMPASS/misc/installation.html>`_.
+For detailed instructions and troubleshooting, see the
+`installation documentation <https://natlabrockies.github.io/COMPASS/misc/installation.html>`_.
 
 
 Quickstart

compass/plugin/ordinance.py

@@ -835,9 +835,11 @@ async def parse_single_doc_for_structured_data(self, extraction_context):
                     doc_for_extraction, out_fn_stem=self.jurisdiction.full_name
                 )
                 extraction_context.attrs["structured_data"] = data_df
+                n_feats = num_ordinances_dataframe(data_df)
+                extraction_context.attrs["num_features_extracted"] = n_feats
                 logger.info(
                     "%d ordinance value(s) found for %s from doc:\n%s. ",
-                    num_ordinances_dataframe(data_df),
+                    n_feats,
                     self.jurisdiction.full_name,
                     doc_for_extraction,
                 )
@@ -897,9 +899,11 @@ async def parse_multi_doc_context_for_structured_data(
         )
 
         extraction_context.attrs["structured_data"] = data_df
+        n_feats = num_ordinances_dataframe(data_df)
+        extraction_context.attrs["num_features_extracted"] = n_feats
         logger.info(
             "%d ordinance value(s) found for %s in %d docs. ",
-            num_ordinances_dataframe(data_df),
+            n_feats,
             self.jurisdiction.full_name,
             extraction_context.num_documents,
         )
@@ -938,6 +942,7 @@ async def parse_multi_doc_concat(self, extraction_context):
         data_dfs = await asyncio.gather(*tasks)
 
         all_data = []
+        total_features = 0
         for doc_ind, (data_df, doc) in enumerate(
             zip(data_dfs, extraction_context, strict=True), start=1
         ):
@@ -950,9 +955,11 @@ async def parse_multi_doc_concat(self, extraction_context):
             await extraction_context.mark_doc_as_data_source(
                 doc, out_fn_stem=f"{self.jurisdiction.full_name}_{doc_ind}"
             )
+            num_found = num_ordinances_dataframe(data_df)
+            total_features += num_found
             logger.info(
                 "%d ordinance value(s) found for %s from doc:\n%s. ",
-                num_ordinances_dataframe(data_df),
+                num_found,
                 self.jurisdiction.full_name,
                 doc,
             )
@@ -968,6 +975,7 @@ async def parse_multi_doc_concat(self, extraction_context):
         extraction_context.attrs["structured_data"] = pd.concat(
             all_data, ignore_index=True
         )
+        extraction_context.attrs["num_features_extracted"] = total_features
         return extraction_context
 
     async def parse_multi_doc_merge(self, extraction_context):

compass/services/openai.py

@@ -218,6 +218,7 @@ def _update_pb_cost(self, response):
             openai.RateLimitError,
             openai.APITimeoutError,
             openai.BadRequestError,
+            openai.APIConnectionError,
         ),
     )
     async def _call_gpt(self, **kwargs):

compass/services/threaded.py

@@ -645,20 +645,28 @@ def _compile_doc_info(doc):
     out_fp = doc.attrs.get("source_fp", doc.attrs.get("out_fp"))
     return {
         "source": doc.attrs.get("source"),
+        "num_features_extracted": doc.attrs.get("num_features_extracted"),
         "effective_year": year if year is not None and year > 0 else None,
         "effective_month": month if month is not None and month > 0 else None,
         "effective_day": day if day is not None and day > 0 else None,
         "ord_filename": Path(out_fp or "unknown").name,
+        "doc_type": doc.attrs.get("doc_type"),
         "num_pages": doc.attrs.get("num_pages", len(doc.pages)),
         "checksum": doc.attrs.get("checksum"),
         "is_pdf": is_pdf_doc(doc),
         "from_ocr": doc.attrs.get("from_ocr", False),
+        "conversion_time_seconds": doc.attrs.get("conversion_time_seconds"),
+        "conversion_status": doc.attrs.get("conversion_status"),
+        "mean_confidence": doc.attrs.get("mean_confidence"),
+        "low_score_confidence": doc.attrs.get("low_score_confidence"),
+        "collection_step_rank": doc.attrs.get("collection_step_rank"),
         "relevant_text_ngram_score": doc.attrs.get(
             "relevant_text_ngram_score"
         ),
         "permitted_use_text_ngram_score": doc.attrs.get(
             "permitted_use_text_ngram_score"
         ),
+        "from_steps": doc.attrs.get("from_steps"),
     }