diff --git a/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/data_retrieval/poe_ninja_currency_api_handler.py b/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/data_retrieval/currency_api_handler.py similarity index 50% rename from src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/data_retrieval/poe_ninja_currency_api_handler.py rename to src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/data_retrieval/currency_api_handler.py index 08b89b270..c78fbb70b 100644 --- a/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/data_retrieval/poe_ninja_currency_api_handler.py +++ b/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/data_retrieval/currency_api_handler.py @@ -4,27 +4,10 @@ from data_retrieval_app.logs.logger import external_data_retrieval_logger as logger -class PoENinjaCurrencyAPIHandler: +class CurrencyAPIHandler: def __init__(self, url: str) -> None: self.url = url - def _combine_currency_data(self, currencies: list, currency_details: list) -> list: - """ - Combines the currency data. - """ - currencies_df = self._json_to_df(currencies) - currency_details_df = self._json_to_df(currency_details) - - combined_currency_data_df = currencies_df.merge( - currency_details_df, - how="left", - # left_on="pay.pay_currency_id", - # right_on="id", - left_on="currencyTypeName", - right_on="name", - ) - return combined_currency_data_df - def _json_to_df(self, currencies: list) -> pd.DataFrame: df = pd.json_normalize(currencies) @@ -44,14 +27,11 @@ def make_request(self) -> pd.DataFrame: raise e response_json = response.json() - currencies = response_json["lines"] - currency_details = response_json["currencyDetails"] - - combined_currency_data_df = self._combine_currency_data( - currencies, currency_details - ) + # items_df = pd.DataFrame(response_json["items"]) + items_df = pd.json_normalize(response_json["items"]) + currency_df = items_df[items_df["category"] == "currency"] - return combined_currency_data_df + return currency_df def store_data_to_csv(self, path: str) -> None: """ @@ -59,4 +39,4 @@ def store_data_to_csv(self, path: str) -> None: """ currencies_df = self.make_request() - currencies_df.to_csv(path + "/poe_ninja_currencies.csv", index=False) + currencies_df.to_csv(path + "/currencies.csv", index=False) diff --git a/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/main.py b/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/main.py index 55fd3debe..5629a7a03 100644 --- a/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/main.py +++ b/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/main.py @@ -11,19 +11,19 @@ import requests from data_retrieval_app.external_data_retrieval.config import settings +from data_retrieval_app.external_data_retrieval.data_retrieval.currency_api_handler import ( + CurrencyAPIHandler, +) from data_retrieval_app.external_data_retrieval.data_retrieval.poe_api_handler import ( PoEAPIHandler, ) -from data_retrieval_app.external_data_retrieval.data_retrieval.poe_ninja_currency_api_handler import ( - PoENinjaCurrencyAPIHandler, +from data_retrieval_app.external_data_retrieval.transforming_data.transform_currency_api_data import ( + TransformCurrencyAPIData, ) from data_retrieval_app.external_data_retrieval.transforming_data.transform_poe_api_data import ( PoEAPIDataTransformerBase, UniquePoEAPIDataTransformer, ) -from data_retrieval_app.external_data_retrieval.transforming_data.transform_poe_ninja_currency_api_data import ( - TransformPoENinjaCurrencyAPIData, -) from data_retrieval_app.external_data_retrieval.utils import ( ProgramRunTooLongException, ProgramTooSlowException, @@ -62,10 +62,10 @@ def __init__( n_unique_wanted_items=10, ) - self.poe_ninja_currency_api_handler = PoENinjaCurrencyAPIHandler( - url=f"https://poe.ninja/api/data/currencyoverview?league={self.current_league}&type=Currency" + self.currency_api_handler = CurrencyAPIHandler( + url=f"https://api.poe.watch/exchange/ratios?league={self.current_league}&game=poe1" ) - self.poe_ninja_transformer = TransformPoENinjaCurrencyAPIData() + self.currency_transformer = TransformCurrencyAPIData() def _get_modifiers(self) -> dict[str, pd.DataFrame]: response = requests.get(self.modifier_url, headers=self.pom_auth_headers) @@ -147,8 +147,8 @@ def _categorize_new_items(self, df: pd.DataFrame) -> dict[str, pd.DataFrame]: return split_dfs def _get_new_currency_data(self) -> pd.DataFrame: - currency_df = self.poe_ninja_currency_api_handler.make_request() - currency_df = self.poe_ninja_transformer.transform_into_tables(currency_df) + currency_df = self.currency_api_handler.make_request() + currency_df = self.currency_transformer.transform_into_tables(currency_df) return currency_df def _initialize_data_stream_threads( @@ -233,6 +233,8 @@ def retrieve_data(self): ) futures[follow_future] = "data_processing" elif future_job == "listener": + logger.exception(crashed_future.exception().with_traceback()) + raise crashed_future.exception() new_future = self._initialize_data_stream_threads( executor, listeners=1, diff --git a/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/transforming_data/transform_poe_ninja_currency_api_data.py b/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/transforming_data/transform_currency_api_data.py similarity index 64% rename from src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/transforming_data/transform_poe_ninja_currency_api_data.py rename to src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/transforming_data/transform_currency_api_data.py index a304a110f..89222809d 100644 --- a/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/transforming_data/transform_poe_ninja_currency_api_data.py +++ b/src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/transforming_data/transform_currency_api_data.py @@ -2,8 +2,8 @@ import requests from data_retrieval_app.external_data_retrieval.config import settings -from data_retrieval_app.external_data_retrieval.data_retrieval.poe_ninja_currency_api_handler import ( - PoENinjaCurrencyAPIHandler, +from data_retrieval_app.external_data_retrieval.data_retrieval.currency_api_handler import ( + CurrencyAPIHandler, ) from data_retrieval_app.logs.logger import transform_logger as logger from data_retrieval_app.pom_api_authentication import get_superuser_token_headers @@ -14,7 +14,7 @@ def load_currency_data(): """ Loads data from the poe.ninja currency API. """ - poe_ninja_currency_api_handler = PoENinjaCurrencyAPIHandler( + poe_ninja_currency_api_handler = CurrencyAPIHandler( url=f"https://poe.ninja/api/data/currencyoverview?league={settings.CURRENT_SOFTCORE_LEAGUE}&type=Currency" ) @@ -23,27 +23,46 @@ def load_currency_data(): return currencies_df -class TransformPoENinjaCurrencyAPIData: +class TransformCurrencyAPIData: def __init__(self) -> None: - logger.debug("Initializing TransformPoENinjaCurrencyAPIData.") + logger.debug("Initializing TransformCurrencyAPIData.") self.base_url = settings.BACKEND_BASE_URL logger.debug(f"Url set to: {self.base_url}") self.pom_api_headers = get_superuser_token_headers(self.base_url) logger.debug("Headers set to: " + str(self.pom_api_headers)) - logger.debug("Initializing TransformPoENinjaCurrencyAPIData done.") + logger.debug("Initializing TransformCurrencyAPIData done.") - def _create_currency_table(self, currency_df: pd.DataFrame) -> pd.DataFrame: + self.name_to_trade_name = self._get_name_to_trade_name_dict() + + def _get_name_to_trade_name_dict(self) -> dict: """ - Creates the currency table. + Retrieves a map for "fancy" currency names, as used in the API, to their trade names, which we need. """ - currency_df.rename( - columns={ - "tradeId": "tradeName", - "chaosEquivalent": "valueInChaos", - }, - inplace=True, - ) - return currency_df + headers = { + "User-Agent": f"OAuth pathofmodifiers/0.1.0 (contact: {settings.OATH_ACC_TOKEN_CONTACT_EMAIL}) StrictMode" + } + try: + response = requests.get( + "https://www.pathofexile.com/api/trade/data/static", headers=headers + ) + response.raise_for_status() + except Exception as e: + logger.error( + f"The following error occurred while making request _get_latest_change_id: {e}" + ) + raise e + + response_json = response.json() + result = response_json["result"] + currencies = {} + for category in result: + if category["id"] == "Currency": + for entry in category["entries"]: + name = entry["text"] + trade_name = entry["id"] + currencies[name] = trade_name + + return currencies def _transform_currency_table( self, currency_df: pd.DataFrame, hours_since_launch: int @@ -51,13 +70,24 @@ def _transform_currency_table( """ Since a chaos orb is always worth one chaos orb, ninja does not include it in its price api. """ + + currency_df["chaos.chaosValue"] = currency_df["chaos.chaosValue"].where( + (currency_df["chaos.chaosValue"] == 0) + | (currency_df["chaos.chaosValue"].isna()), + currency_df["divine.chaosValue"], + ) + chaos_dict = { - "tradeName": ["chaos"], - "valueInChaos": [1], + "name": ["Chaos Orb"], + "chaos.chaosValue": [1], } chaos_df = pd.DataFrame.from_dict(chaos_dict) currency_df = pd.concat((currency_df, chaos_df), ignore_index=True) + currency_df["tradeName"] = currency_df["name"].map( + lambda name: self.name_to_trade_name.get(name, pd.NA) + ) + currency_df["createdHoursSinceLaunch"] = hours_since_launch return currency_df @@ -65,15 +95,17 @@ def _clean_currency_table(self, currency_df: pd.DataFrame) -> pd.DataFrame: """ Cleans the currency table of unnecessary columns. """ + currency_df = currency_df.rename(columns={"chaos.chaosValue": "valueInChaos"}) - currency_df.drop( + currency_df = currency_df.drop( currency_df.columns.difference( ["tradeName", "valueInChaos", "createdHoursSinceLaunch"] ), axis=1, - inplace=True, ) - currency_df = currency_df.loc[~currency_df["tradeName"].isna()].reset_index() + currency_df = currency_df.loc[~currency_df["tradeName"].isna()].reset_index( + drop=True + ) return currency_df def _get_latest_currency_id_series(self, currency_df: pd.DataFrame) -> pd.Series: @@ -102,7 +134,6 @@ def transform_into_tables(self, currency_df: pd.DataFrame) -> pd.DataFrame: """ hours_since_launch = find_hours_since_launch() logger.debug("Transforming data into tables.") - currency_df = self._create_currency_table(currency_df) currency_df = self._transform_currency_table(currency_df, hours_since_launch) logger.debug("Successfully transformed data into tables.") diff --git a/src/backend_data_retrieval/data_retrieval_app/tests/external_data_retrieval/data_retrieval/test_poe_ninja_currency_api_handler.py b/src/backend_data_retrieval/data_retrieval_app/tests/external_data_retrieval/data_retrieval/test_poe_ninja_currency_api_handler.py index 284760495..c69feef37 100644 --- a/src/backend_data_retrieval/data_retrieval_app/tests/external_data_retrieval/data_retrieval/test_poe_ninja_currency_api_handler.py +++ b/src/backend_data_retrieval/data_retrieval_app/tests/external_data_retrieval/data_retrieval/test_poe_ninja_currency_api_handler.py @@ -1,4 +1,4 @@ -class TestPoENinjaCurrencyAPIHandler: +class TestCurrencyAPIHandler: """ settings.POE_PUBLIC_STASHES_AUTH_TOKEN and settings.OATH_ACC_TOKEN_CONTACT_EMAIL needs to be set to for this test to work diff --git a/src/backend_data_retrieval/data_retrieval_app/tests/external_data_retrieval/transforming_data/transformation_models/test_transform_poe_ninja_currency_api_data.py b/src/backend_data_retrieval/data_retrieval_app/tests/external_data_retrieval/transforming_data/transformation_models/test_transform_poe_ninja_currency_api_data.py index 22e3d1645..e2edc7edf 100644 --- a/src/backend_data_retrieval/data_retrieval_app/tests/external_data_retrieval/transforming_data/transformation_models/test_transform_poe_ninja_currency_api_data.py +++ b/src/backend_data_retrieval/data_retrieval_app/tests/external_data_retrieval/transforming_data/transformation_models/test_transform_poe_ninja_currency_api_data.py @@ -1,3 +1,3 @@ -class TestTransformPoeNinjaCurrencyAPIData: +class TestTransformCurrencyAPIData: # TODO: Add tests if needed pass