From 77a75b1b34215f5f0e53fa287dd73c59c1fc8b4a Mon Sep 17 00:00:00 2001 From: Michael Flanakin Date: Wed, 6 May 2026 08:07:02 -0700 Subject: [PATCH 1/6] Add FOCUS 1.3 ingestion (#2124) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds IngestionSetup_v1_3.kql with Costs/Prices/CommitmentDiscountUsage/ Recommendations/Transactions transforms and final tables renamed for FOCUS 1.3. Cost and Usage gains 8 new FOCUS 1.3 columns: AllocatedMethodId/Details/ResourceId/ResourceName/Tags (data-generator split cost allocation), ContractApplied (per-row contract commitment application), ServiceProviderName + HostProviderName (replacing the deprecated ProviderName/PublisherName, with empty-fallback for back compat). Costs_raw now carries the new columns so downstream v1_2 transforms keep working and v1_3 transforms can read them directly. The v1_3 file is wired into the Bicep deployment alongside v1_0 and v1_2. The unversioned Costs() function still aliases to v1_2 — phase 2 adds Costs_v1_3() and retargets Latest. Co-Authored-By: Claude Opus 4.7 (1M context) --- src/templates/finops-hub/.build.config | 3 +- .../Microsoft.FinOpsHubs/Analytics/app.bicep | 1 + .../scripts/IngestionSetup_RawTables.kql | 8 + .../Analytics/scripts/IngestionSetup_v1_3.kql | 1966 +++++++++++++++++ 4 files changed, 1977 insertions(+), 1 deletion(-) create mode 100644 src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_v1_3.kql diff --git a/src/templates/finops-hub/.build.config b/src/templates/finops-hub/.build.config index 2dd495e60..0963fb494 100644 --- a/src/templates/finops-hub/.build.config +++ b/src/templates/finops-hub/.build.config @@ -33,7 +33,8 @@ "modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_HubInfra.kql", "modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_RawTables.kql", "modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_v1_0.kql", - "modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_v1_2.kql" + "modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_v1_2.kql", + "modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_v1_3.kql" ] }, { diff --git a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/app.bicep b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/app.bicep index b880e7108..42cd2738b 100644 --- a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/app.bicep +++ b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/app.bicep @@ -378,6 +378,7 @@ module ingestion_VersionedScripts '../../fx/hub-database.bicep' = if (useAzure) scripts: { v1_0: loadTextContent('scripts/IngestionSetup_v1_0.kql') v1_2: loadTextContent('scripts/IngestionSetup_v1_2.kql') + v1_3: loadTextContent('scripts/IngestionSetup_v1_3.kql') } continueOnErrors: continueOnErrors forceUpdateTag: forceUpdateTag diff --git a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_RawTables.kql b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_RawTables.kql index 94fead987..83a3be40e 100644 --- a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_RawTables.kql +++ b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_RawTables.kql @@ -359,6 +359,11 @@ // Costs_raw table -- Redefine all columns .alter table Costs_raw ( + AllocatedMethodDetails: string, // FOCUS 1.3+ (JSON) + AllocatedMethodId: string, // FOCUS 1.3+ + AllocatedResourceId: string, // FOCUS 1.3+ + AllocatedResourceName: string, // FOCUS 1.3+ + AllocatedTags: string, // FOCUS 1.3+ (JSON) AvailabilityZone: string, // FOCUS 0.5+ BilledCost: real, // FOCUS 0.5+ BillingAccountId: string, // FOCUS 0.5+ @@ -385,9 +390,11 @@ CommitmentDiscountUnit: string, // FOCUS 1.1+ ConsumedQuantity: real, // FOCUS 1.0+ ConsumedUnit: string, // FOCUS 1.0+ + ContractApplied: string, // FOCUS 1.3+ (JSON) ContractedCost: real, // FOCUS 1.0+ ContractedUnitPrice: real, // FOCUS 1.0+ EffectiveCost: real, // FOCUS 1.0-preview+ + HostProviderName: string, // FOCUS 1.3+ InvoiceId: string, // FOCUS 1.2+ InvoiceIssuerName: string, // FOCUS 0.5+ ListCost: real, // FOCUS 1.0-preview+ @@ -406,6 +413,7 @@ ResourceType: string, // FOCUS 1.0-preview+ ServiceCategory: string, // FOCUS 0.5+ ServiceName: string, // FOCUS 0.5+ + ServiceProviderName: string, // FOCUS 1.3+ ServiceSubcategory: string, // FOCUS 1.1+ SkuId: string, // FOCUS 1.0-preview+ SkuMeter: string, // FOCUS 1.1+ diff --git a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_v1_3.kql b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_v1_3.kql new file mode 100644 index 000000000..70d29a2ed --- /dev/null +++ b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_v1_3.kql @@ -0,0 +1,1966 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//====================================================================================================================== +// Ingestion database +// See known issues @ https://github.com/microsoft/finops-toolkit/issues/1111 +//====================================================================================================================== + +// For allowed commands, see https://learn.microsoft.com/azure/data-explorer/database-script + +//===| Prices |========================================================================================================= +// Supported versions: +// - MS EA 2023-05-01 -- See https://learn.microsoft.com/azure/cost-management-billing/dataset-schema/price-sheet-ea +// - MS MCA 2023-05-01 -- See https://learn.microsoft.com/azure/cost-management-billing/dataset-schema/price-sheet-mca +//====================================================================================================================== + +// Prices_transform_v1_3 function +.create-or-alter function +with (docstring='Transforms Prices_raw into FOCUS 1.3.', folder='Prices') +Prices_transform_v1_3() +{ + // NOTE: All open issues and questions are tracked @ https://github.com/microsoft/finops-toolkit/issues/1111 + let isoMonths = (duration: string) { + let number = toint(replace_regex(duration, @'[PMY]', '')); + toint(case( + duration == '', int(null), + duration endswith "Y", number * 12, + duration endswith "M", number, + -1 + )) + }; + let prices = materialize( + Prices_raw + | extend PricingCurrency = coalesce(Currency, CurrencyCode) // CurrencyCode last as a fallback only + | extend x_SkuId = coalesce(SkuId, SkuID) + | extend x_SkuMeterId = coalesce(MeterId, MeterID) + | extend x_SkuProductId = coalesce(ProductId, ProductID) + | extend x_SkuTerm = isoMonths(Term) + | project-rename + SkuMeter = MeterName, + x_BaseUnitPrice = BasePrice, + x_EffectivePeriodEnd = EffectiveEndDate, + x_EffectivePeriodStart = EffectiveStartDate, + x_PricingUnitDescription = UnitOfMeasure, + x_SkuIncludedQuantity = IncludedQuantity, + x_SkuMeterCategory = MeterCategory, + x_SkuMeterSubcategory = MeterSubCategory, + x_SkuMeterType = MeterType, + x_SkuOfferId = OfferID, + x_SkuPartNumber = PartNumber, + x_SkuPriceType = PriceType, + x_SkuRegion = MeterRegion, + x_SkuServiceFamily = ServiceFamily, + x_SkuTier = TierMinimumUnits + | extend ContractedUnitPrice = iff(x_SkuPriceType != 'SavingsPlan', UnitPrice, real(null)) // UnitPrice for savings plan is not the on-demand unit price + | extend ListUnitPrice = iff(x_SkuPriceType != 'SavingsPlan', MarketPrice, real(null)) // MarketPrice for savings plan is not the list price + | extend ChargeCategory = case( + x_SkuPriceType == 'Consumption', 'Usage', + x_SkuPriceType == 'ReservedInstance', 'Purchase', + x_SkuPriceType == 'SavingsPlan', 'Usage', // Savings plan prices are for committed usage, not the purchase + '' + ) + | extend SkuPriceIdv2 = strcat(case(x_SkuPriceType == 'Consumption', 'OD', x_SkuPriceType == 'ReservedInstance', 'RI', x_SkuPriceType == 'SavingsPlan', 'SP', 'XX'), substring(ChargeCategory, 0, 1), x_SkuTerm, '_', x_SkuProductId, '_', x_SkuId, '_', x_SkuMeterType, '_', x_SkuTier, x_SkuOfferId) + | extend x_BillingAccountId = iff(BillingAccountId startswith '/', split(BillingAccountId, '/')[-1], coalesce(BillingAccountId, EnrollmentNumber)) + | extend x_BillingProfileId = iff(BillingProfileId startswith '/', split(BillingProfileId, '/')[-1], coalesce(BillingProfileId, EnrollmentNumber)) + | extend tmp_SavingsPlanKey = strcat(x_SkuMeterId, x_SkuProductId, x_SkuId, x_SkuTier, x_SkuOfferId) + // + // Get latest ingested row based on the unique ID + | extend x_IngestionTime = ingestion_time() + ); + // + // Meters for reservations and savings plans to identify commitment eligibility + let riMeters = prices | where x_SkuPriceType == 'ReservedInstance' | distinct x_SkuMeterId; + let spMeters = prices | where x_SkuPriceType == 'SavingsPlan' | distinct x_SkuMeterId; + // + // Copy list/base/contracted prices from on-demand SKUs + prices + | where x_SkuPriceType == 'SavingsPlan' + // If we use join, specify the shuffle key + // TODO: Compare join vs. lookup perf -- | join kind=leftouter hint.strategy=shuffle (prices | where x_SkuPriceType == 'Consumption' | where x_SkuMeterId in (spMeters) | distinct tmp_SavingsPlanKey, ListUnitPrice, ContractedUnitPrice, x_BaseUnitPrice) on tmp_SavingsPlanKey + | lookup kind=leftouter (prices | where x_SkuPriceType == 'Consumption' | where x_SkuMeterId in (spMeters) | distinct tmp_SavingsPlanKey, ListUnitPrice, ContractedUnitPrice, x_BaseUnitPrice) on tmp_SavingsPlanKey + | extend ListUnitPrice = coalesce(ListUnitPrice, ListUnitPrice1) + | extend ContractedUnitPrice = coalesce(ContractedUnitPrice, ContractedUnitPrice1) + | extend x_BaseUnitPrice = coalesce(x_BaseUnitPrice, x_BaseUnitPrice1) + | project-away ListUnitPrice1, ContractedUnitPrice1, x_BaseUnitPrice1, tmp_SavingsPlanKey + | union ((prices | where x_SkuPriceType != 'SavingsPlan')) + // + // Set CommitmentDiscountCategory for reuse + | extend CommitmentDiscountCategory = case( + x_SkuPriceType == 'ReservedInstance', 'Usage', + x_SkuPriceType == 'SavingsPlan', 'Spend', + '' + ) + // + // Calculate commitment discount eligibility + // TODO: Would a join be faster? + // TODO: Check this to ensure it's correct + | extend x_CommitmentDiscountSpendEligibility = iff(x_SkuMeterId in (riMeters) and x_SkuPriceType != 'ReservedInstance', 'Eligible', 'Not Eligible') + | extend x_CommitmentDiscountUsageEligibility = iff(x_SkuMeterId in (spMeters), 'Eligible', 'Not Eligible') + // + // TODO: Implement x_CommitmentDiscountNormalizedRatio + | extend x_CommitmentDiscountNormalizedRatio = real(null) + // + // Add PricingUnit and x_PricingBlockSize + // TODO: Compare join vs. lookup perf -- | join kind=leftouter (PricingUnits) on x_PricingUnitDescription | project-away x_PricingUnitDescription1 + | lookup kind=leftouter (PricingUnits) on x_PricingUnitDescription + // + | extend x_EffectiveUnitPrice = iff(x_SkuPriceType == 'SavingsPlan', UnitPrice, real(null)) // Savings plan prices are for the effective price, not the contracted price + | extend x_EffectiveUnitPriceDiscount = ContractedUnitPrice - x_EffectiveUnitPrice + | extend x_ContractedUnitPriceDiscount = ListUnitPrice - ContractedUnitPrice + | extend x_TotalUnitPriceDiscount = ListUnitPrice - x_EffectiveUnitPrice + | project + BillingAccountId = tolower(case( + BillingProfileId startswith '/', BillingProfileId, + BillingAccountId startswith '/', BillingAccountId, + strcat('/providers/microsoft.billing/billingaccounts/', x_BillingAccountId, iff(x_BillingProfileId == x_BillingAccountId, '', strcat('/billingprofiles/', x_BillingProfileId))) + )), + BillingAccountName = coalesce(BillingProfileName, BillingAccountName, x_BillingProfileId), + BillingCurrency = coalesce(BillingCurrency, CurrencyCode, Currency), // Currency last as a fallback only + ChargeCategory, + CommitmentDiscountCategory, + CommitmentDiscountType = case( + x_SkuPriceType == 'ReservedInstance', 'Reservation', + x_SkuPriceType == 'SavingsPlan', 'Savings plan', + '' + ), + CommitmentDiscountUnit = case( + isempty(CommitmentDiscountCategory), '', + CommitmentDiscountCategory == 'Spend', PricingCurrency, + CommitmentDiscountCategory == 'Usage' and x_CommitmentDiscountNormalizedRatio == real(1), PricingUnit, + CommitmentDiscountCategory == 'Usage' and x_CommitmentDiscountNormalizedRatio > real(1), strcat('Normalized ', PricingUnit), + '' + ), + ContractedUnitPrice, + ListUnitPrice, + PricingCategory = case( + x_SkuPriceType == 'Consumption', 'Standard', + x_SkuPriceType == 'ReservedInstance', 'Standard', // Reservation purchases are tracked as "Standard" + x_SkuPriceType == 'SavingsPlan', 'Committed', + '' + ), + PricingCurrency, + PricingUnit, + SkuId = coalesce(ProductId, ProductID), + SkuMeter, + SkuPriceId = strcat(x_SkuProductId, '_', x_SkuId, '_', x_SkuMeterType), + SkuPriceIdv2, + x_BaseUnitPrice, + x_BillingAccountAgreement = case( + strlen(x_BillingAccountId) > 32, 'MCA', + strlen(x_BillingAccountId) < 32, 'EA', + 'Unknown' + ), + x_BillingAccountId, + x_BillingProfileId, + x_CommitmentDiscountNormalizedRatio, + x_CommitmentDiscountSpendEligibility, + x_CommitmentDiscountUsageEligibility, + x_ContractedUnitPriceDiscount, + x_ContractedUnitPriceDiscountPercent = 1.0 * x_ContractedUnitPriceDiscount / ListUnitPrice * 100, + x_EffectivePeriodEnd = startofmonth(x_EffectivePeriodEnd + 1h), + x_EffectivePeriodStart, + x_EffectiveUnitPrice, + x_EffectiveUnitPriceDiscount, + x_EffectiveUnitPriceDiscountPercent = 1.0 * x_EffectiveUnitPriceDiscount / ContractedUnitPrice * 100, + x_IngestionTime, + x_PricingBlockSize, + x_PricingSubcategory = case( + x_SkuPriceType == 'Consumption' and (x_SkuIncludedQuantity > 0 or x_SkuTier > 0), 'Tiered', + x_SkuPriceType == 'Consumption', 'Standard', + x_SkuPriceType == 'ReservedInstance', 'Standard', // Reservation purchases are tracked as "Standard" + x_SkuPriceType == 'SavingsPlan', 'Committed Spend', + '' + ), + x_PricingUnitDescription, + x_SkuDescription = Product, + x_SkuId, + x_SkuIncludedQuantity, + x_SkuMeterCategory, + x_SkuMeterId, + x_SkuMeterSubcategory, + x_SkuMeterType, + x_SkuPriceType, + x_SkuProductId, + x_SkuRegion, + x_SkuServiceFamily, + x_SkuOfferId, + x_SkuPartNumber, + x_SkuTerm, + x_SkuTier, + x_SourceName = coalesce(x_SourceName, 'Cost Management'), + x_SourceProvider = coalesce(x_SourceProvider, 'Microsoft'), + x_SourceType = coalesce(x_SourceType, 'PriceSheet'), + x_SourceVersion = coalesce(x_SourceVersion, '2023-05-01'), + x_TotalUnitPriceDiscount, + x_TotalUnitPriceDiscountPercent = 1.0 * x_TotalUnitPriceDiscount / ListUnitPrice * 100 +} + +// Prices_final_v1_3 table +.create-merge table Prices_final_v1_3 ( + BillingAccountId: string, + BillingAccountName: string, + BillingCurrency: string, + ChargeCategory: string, + CommitmentDiscountCategory: string, + CommitmentDiscountType: string, + CommitmentDiscountUnit: string, + ContractedUnitPrice: real, + ListUnitPrice: real, + PricingCategory: string, + PricingCurrency: string, // Azure + PricingUnit: string, + SkuId: string, + SkuMeter: string, // Azure + SkuPriceId: string, + SkuPriceIdv2: string, // Hubs add-on + x_BaseUnitPrice: real, // Azure + x_BillingAccountAgreement: string, // Hubs add-on + x_BillingAccountId: string, // Azure MCA + x_BillingProfileId: string, // Azure MCA + x_CommitmentDiscountNormalizedRatio: real, // Hubs add-on + x_CommitmentDiscountSpendEligibility: string, // Hubs add-on + x_CommitmentDiscountUsageEligibility: string, // Hubs add-on + x_ContractedUnitPriceDiscount: real, // Hubs add-on + x_ContractedUnitPriceDiscountPercent: real, // Hubs add-on + x_EffectivePeriodEnd: datetime, // Azure + x_EffectivePeriodStart: datetime, // Azure + x_EffectiveUnitPrice: real, // Azure + x_EffectiveUnitPriceDiscount: real, // Hubs add-on + x_EffectiveUnitPriceDiscountPercent: real, // Hubs add-on + x_IngestionTime: datetime, // Hubs add-on + x_PricingBlockSize: real, // Hubs add-on + x_PricingSubcategory: string, // Hubs add-on + x_PricingUnitDescription: string, // Azure + x_SkuDescription: string, // Azure + x_SkuId: string, // Azure + x_SkuIncludedQuantity: real, // Azure EA + x_SkuMeterCategory: string, // Azure + x_SkuMeterId: string, // Azure + x_SkuMeterSubcategory: string, // Azure + x_SkuMeterType: string, // Azure + x_SkuPriceType: string, // Azure + x_SkuProductId: string, // Azure + x_SkuRegion: string, // Azure + x_SkuServiceFamily: string, // Azure + x_SkuOfferId: string, // Azure EA + x_SkuPartNumber: string, // Azure EA + x_SkuTerm: int, // Azure + x_SkuTier: real, // Azure MCA + x_SourceName: string, // Hubs add-on + x_SourceProvider: string, // Hubs add-on + x_SourceType: string, // Hubs add-on + x_SourceVersion: string, // Hubs add-on + x_TotalUnitPriceDiscount: real, // Hubs add-on + x_TotalUnitPriceDiscountPercent: real // Hubs add-on +) + +// Update policy for Prices_raw -> Prices_final_v1_3 +.alter table Prices_final_v1_3 policy update +``` +[{ + "IsEnabled": true, + "Source": "Prices_raw", + "Query": "Prices_transform_v1_3()", + "IsTransactional": true, + "PropagateIngestionProperties": true +}] +``` + + +//===| Cost and usage |================================================================================================= +// Supported versions: +// - MS: 1.2-preview, 1.0, 1.0-preview(v1) +// https://aka.ms/costmgmt/exports/focus +// - AWS: 1.0 +// https://docs.aws.amazon.com/cur/latest/userguide/table-dictionary-focus-1-0-aws-columns.html +// - GCP: Jan-Jun 2024 +// https://cloud.google.com/resources/google-cloud-focus?e=48754805&hl=en +// Links to (Aug 2024): https://services.google.com/fh/files/misc/focus_guide_v1.pdf +// See also: +// - https://cloud.google.com/billing/docs/how-to/export-data-bigquery-tables/standard-usage +// - https://cloud.google.com/billing/docs/how-to/export-data-bigquery-tables/detailed-usage +// - OCI: 1.0 +// https://docs.oracle.com/iaas/Content/Billing/Concepts/costusagereportsoverview.htm#costreports__focus-cost-report-schema +// +// Support for non-Azure data is limited to ingestion only. Data is not transformed across versions. +//====================================================================================================================== + +// Costs_transform_v1_3 function +.create-or-alter function +with (docstring='All costs transformed to FOCUS 1.3.', folder='Costs') +Costs_transform_v1_3() +{ + let checkString = (column: string, oldValue: string, newValue: string) { + iff(oldValue == newValue, dynamic({}), bag_pack(column, oldValue)) + }; + let checkInt = (column: string, oldValue: int, newValue: int) { + iff(oldValue == newValue, dynamic({}), bag_pack(column, oldValue)) + }; + let checkReal = (column: string, oldValue: real, newValue: real) { + iff(oldValue == newValue, dynamic({}), bag_pack(column, oldValue)) + }; + // NOTE: All open issues and questions are tracked @ https://github.com/microsoft/finops-toolkit/issues/1111 + Costs_raw + // + // Dedupe rows + | extend x_IngestionTime = ingestion_time() + | extend x_ChargeId = '' + // TODO: Consider adding a unique charge ID per row + // hash_sha256(strcat( + // // DO NOT CHANGE COLUMNS OR COLUMN ORDER + // // 1. Resource hierarchy (including resource name), highest to lowest + // BillingAccountId, + // x_InvoiceSectionId, + // x_AccountOwnerId, + // SubAccountId, + // x_ResourceGroupName, + // ResourceName, + // // 2. Resource details + // ResourceId, + // RegionId, + // Tags, + // CommitmentDiscountId, + // x_CostCenter, + // // 4. Meter details + // SkuPriceId, + // x_SkuMeterId, + // x_SkuPartNumber, + // x_SkuOfferId, + // x_SkuDetails, + // // 5. Date + // ChargePeriodStart + // )) + // + // Identify data quality issues + // TODO: Remove x_SourceChanges in v1_3 (or later) + | extend x_SourceChanges = trim_end(',', strcat( + iff(ChargeCategory == 'Purchase' and isnotempty(CommitmentDiscountId) and ChargeFrequency == 'Usage-Based', 'InvalidChargeFrequency,', ''), + iff(ChargeCategory == 'Purchase' and isnotempty(CommitmentDiscountId) and EffectiveCost > 0, 'InvalidEffectiveCost,', ''), + iff((isempty(ContractedCost) or ContractedCost == 0) and EffectiveCost != 0, 'MissingContractedCost,', ''), + iff((isempty(ContractedUnitPrice) or ContractedUnitPrice == 0) and x_EffectiveUnitPrice != 0, 'MissingContractedUnitPrice,', ''), + iff(ListCost < ContractedCost, 'ListCostLessThanContractedCost,', ''), + iff(ContractedCost < EffectiveCost, 'ContractedCostLessThanEffectiveCost,', ''), + iff((isempty(ListCost) or ListCost == 0) and (ContractedCost != 0 or EffectiveCost != 0), 'MissingListCost,', ''), + iff((isempty(ListUnitPrice) or ListUnitPrice == 0) and (ContractedUnitPrice != 0 or x_EffectiveUnitPrice != 0), 'MissingListUnitPrice,', ''), + iff((isnotempty(x_BilledUnitPrice) and x_BilledUnitPrice != 0 and isnotempty(x_EffectiveUnitPrice) and x_EffectiveUnitPrice != 0 and abs(x_BilledUnitPrice - x_EffectiveUnitPrice) < 0.0001) + or (isnotempty(ContractedUnitPrice) and ContractedUnitPrice != 0 and isnotempty(x_EffectiveUnitPrice) and x_EffectiveUnitPrice != 0 and abs(ContractedUnitPrice - x_EffectiveUnitPrice) < 0.0001), + 'XEffectiveUnitPriceRoundingError,', ''), + iff(ConsumedQuantity == 0 and (EffectiveCost != 0 or BilledCost != 0), 'MissingConsumedQuantity,', ''), + iff(PricingQuantity == 0 and (EffectiveCost != 0 or BilledCost != 0), 'MissingPricingQuantity,', ''), + iff(isempty(ProviderName), 'MissingProviderName,', ''), + iff(isempty(PublisherName), 'MissingPublisherName,', ''), + iff(ChargeCategory == 'Purchase' and isnotempty(CommitmentDiscountId) and isempty(ResourceId), 'MissingResourceId,', ''), + iff(ChargeCategory == 'Purchase' and isnotempty(CommitmentDiscountId) and isempty(ResourceName), 'MissingResourceName,', ''), + iff(ChargeCategory == 'Purchase' and isnotempty(CommitmentDiscountId) and isempty(ResourceType), 'MissingResourceType,', ''), + iff(BilledCost > 0 and x_BilledUnitPrice == 0, 'MissingXBilledUnitPrice,', ''), + iff(ChargeCategory == 'Purchase' and isnotempty(CommitmentDiscountId) and isempty(x_ResourceType), 'MissingXResourceType,', ''), + iff(PricingCategory == 'Standard' and isnotempty(CommitmentDiscountId) and ChargeCategory == 'Usage', 'PricingCategoryShouldBeCommitted,', ''), + iff(x_SkuTerm == '1Year' or x_SkuTerm == '3Years' or x_SkuTerm == '5Years', 'SkuTermShouldBeAnInteger,', '') + )) + // + // Handle provider columns that moved to FOCUS + | extend PricingCurrency = coalesce(PricingCurrency, x_PricingCurrency) + // + // Backup original prices/costs before the merge + | extend old_ContractedCost = ContractedCost + | extend old_ContractedUnitPrice = ContractedUnitPrice + | extend old_ListCost = ListCost + | extend old_ListUnitPrice = ListUnitPrice + | extend old_x_EffectiveUnitPrice = x_EffectiveUnitPrice + // + // Fix columns needed in other changes + | extend old_ProviderName = ProviderName, ProviderName = case( + isnotempty(ProviderName), ProviderName, + isnotempty(coalesce(x_CostCategories, x_Discount, x_Operation, x_ServiceCode, x_UsageType)), 'AWS', + isnotempty(coalesce(tostring(UsageAmount), tostring(x_Cost), x_Credits, x_CostType, tostring(x_CurrencyConversionRate), tostring(x_ExportTime), x_Project, x_ServiceId)), 'GCP', + isnotempty(coalesce(x_BillingProfileId, x_InvoiceSectionId)), 'Microsoft', + '' + ) + // + // Identify source + | extend x_SourceName = coalesce(x_SourceName, iff(isnotempty(x_BillingProfileId), 'Cost Management', ProviderName)) + | extend x_SourceProvider = coalesce(x_SourceProvider, ProviderName) + | extend x_SourceType = coalesce(x_SourceType, iff(isnotempty(x_BillingProfileId), 'FocusCost', '')) + | extend x_SourceVersion = coalesce(x_SourceVersion, case( + isnotempty(coalesce(InvoiceId, SkuMeter, PricingCurrency)) and isempty(SkuPriceDetails) and isnotempty(x_SkuDetails), '1.2-preview', + isnotempty(coalesce(InvoiceId, SkuMeter, PricingCurrency)), '1.2', + isnotempty(coalesce(ChargeClass, CommitmentDiscountStatus, tostring(ConsumedQuantity), ConsumedUnit, tostring(ContractedCost), tostring(ContractedUnitPrice), RegionId, RegionName)), '1.0', + isnotempty(coalesce(ChargeSubcategory, Region, tostring(UsageQuantity), UsageUnit)), iff(ProviderName == 'Microsoft', '1.0-preview(v1)', '1.0-preview'), + '' + )) + // Append version check error code + | extend x_SourceChanges = iff(x_SourceVersion == '1.0', x_SourceChanges, + strcat(x_SourceChanges, iff(isempty(x_SourceChanges), '', ','), iff(x_SourceVersion == '', 'UnknownFocusVersion', 'LegacyFocusVersion')) + ) + // + // Fix quantities + | extend old_PricingQuantity = PricingQuantity, PricingQuantity = case( + PricingQuantity != 0 or (EffectiveCost == 0 and BilledCost == 0), PricingQuantity, + PricingQuantity == 0 and isnotempty(BilledCost) and BilledCost != 0 and isnotempty(x_BilledUnitPrice) and x_BilledUnitPrice != 0, BilledCost / x_BilledUnitPrice, + PricingQuantity == 0 and isnotempty(BilledCost) and BilledCost != 0 and isnotempty(x_BilledUnitPrice) and x_BilledUnitPrice != 0, BilledCost / x_BilledUnitPrice, + PricingQuantity == 0 and isnotempty(EffectiveCost) and EffectiveCost != 0 and isnotempty(x_EffectiveUnitPrice) and x_EffectiveUnitPrice != 0, EffectiveCost / x_EffectiveUnitPrice, + PricingQuantity + ) + | extend old_ConsumedQuantity = ConsumedQuantity, ConsumedQuantity = case( + isnotempty(ConsumedQuantity) and ConsumedQuantity != 0, ConsumedQuantity, + ChargeCategory == 'Usage', PricingQuantity / coalesce(x_PricingBlockSize, real(1)), + ConsumedQuantity + ) + // + // Populate missing prices -- mapping to on-demand prices requires meter ID and offer ID + | extend tmp_MissingPrices = ProviderName == 'Microsoft' + and (isempty(ListUnitPrice) or isempty(ContractedUnitPrice) or ListUnitPrice == 0 or ContractedUnitPrice == 0) + and x_EffectiveUnitPrice != 0 + and not(CommitmentDiscountCategory == 'Spend' and CommitmentDiscountStatus == 'Unused') + and isnotempty(strcat(x_SkuMeterId, x_SkuOfferId)) + | as allCosts + | where tmp_MissingPrices + | extend tmp_ReservationPriceLookupKey = tolower(strcat(x_BillingProfileId, substring(ChargePeriodStart, 0, 7), x_SkuMeterId, x_SkuOfferId)) + | as costsWithMissingPrices + | join kind=leftouter ( + Prices_final_v1_3 + | extend tmp_ReservationPriceLookupKey = tolower(strcat(x_BillingProfileId, substring(x_EffectivePeriodStart, 0, 7), x_SkuMeterId, x_SkuOfferId)) + | where x_SkuPriceType == 'Consumption' and tmp_ReservationPriceLookupKey in ((costsWithMissingPrices | summarize by tmp_ReservationPriceLookupKey)) + | summarize ListUnitPrice = min(ListUnitPrice), ContractedUnitPrice = min(ContractedUnitPrice) by tmp_ReservationPriceLookupKey, x_PricingBlockSize, PricingUnit + ) on tmp_ReservationPriceLookupKey + // + // Select the best price to use for each row + | extend x_EffectiveUnitPrice = case( + // If price is a rounding error away from the billed price, use the billed price + isnotempty(x_BilledUnitPrice) and x_BilledUnitPrice != 0 and isnotempty(x_EffectiveUnitPrice) and x_EffectiveUnitPrice != 0 and abs(x_BilledUnitPrice - x_EffectiveUnitPrice) < 0.0001, x_BilledUnitPrice, + // If price is a rounding error away from the contracted price, use the contracted price + isnotempty(ContractedUnitPrice) and ContractedUnitPrice != 0 and isnotempty(x_EffectiveUnitPrice) and x_EffectiveUnitPrice != 0 and abs(ContractedUnitPrice - x_EffectiveUnitPrice) < 0.0001, ContractedUnitPrice, + x_EffectiveUnitPrice + ) + | extend ContractedUnitPrice = case( + // If price is already correct, keep that + (isnotempty(ContractedUnitPrice) and ContractedUnitPrice != 0) or (EffectiveCost == 0 and BilledCost == 0), ContractedUnitPrice, + // If both prices use the same scale, use the new one + PricingUnit == PricingUnit1 and x_PricingBlockSize == x_PricingBlockSize1, ContractedUnitPrice1 * x_BillingExchangeRate, + // If prices are the same unit but not the same scale, use the new one but correct the scale + PricingUnit == PricingUnit1 and x_PricingBlockSize != x_PricingBlockSize1 and isnotempty(x_PricingBlockSize) and isnotempty(x_PricingBlockSize1), ContractedUnitPrice1 * x_BillingExchangeRate / x_PricingBlockSize1 * x_PricingBlockSize, + // If billed price is available, assume the billed price is the same as contracted price to support aggregations + isnotempty(x_BilledUnitPrice) and x_BilledUnitPrice != 0, x_EffectiveUnitPrice, + // Otherwise, assume the effective price is the same as contracted price to support aggregations + x_EffectiveUnitPrice + ) + | extend ListUnitPrice = case( + // If price is already correct, keep that + (isnotempty(ListUnitPrice) and ListUnitPrice != 0) or (EffectiveCost == 0 and BilledCost == 0), ListUnitPrice, + // If both prices use the same scale, use the new one + PricingUnit == PricingUnit1 and x_PricingBlockSize == x_PricingBlockSize1, ListUnitPrice1 * x_BillingExchangeRate, + // If prices are the same unit but not the same scale, use the new one but correct the scale + PricingUnit == PricingUnit1 and x_PricingBlockSize != x_PricingBlockSize1 and isnotempty(x_PricingBlockSize) and isnotempty(x_PricingBlockSize1), ListUnitPrice1 * x_BillingExchangeRate / x_PricingBlockSize1 * x_PricingBlockSize, + // Otherwise, assume the contracted price is the same as list price to support aggregations + ContractedUnitPrice + ) + // Calculate missing costs based on new prices -- If cost is already correct, keep that; if not and price is available, recalculate the cost; otherwise, keep the existing cost + | extend ContractedCost = case( + // If not set or there's no cost, keep the original value + (isnotempty(ContractedCost) and ContractedCost != 0) or (EffectiveCost == 0 and BilledCost == 0), ContractedCost, + // ContractedCost is 0 in all other scenarios... + // If 0 and there's a billed cost and prices are the same, use BilledCost + isnotempty(BilledCost) and BilledCost != 0 and ContractedUnitPrice == x_BilledUnitPrice, BilledCost, + // If 0 and there's a billed cost and prices are the same, use EffectiveCost + isnotempty(EffectiveCost) and EffectiveCost != 0 and ContractedUnitPrice == x_EffectiveUnitPrice, EffectiveCost, + // If 0 and there's a price, calculate the cost based on the price + isnotempty(ContractedUnitPrice) and ContractedUnitPrice != 0, ContractedUnitPrice * PricingQuantity, + // If 0 and there's no price, assume EffectiveCost + isempty(ContractedUnitPrice) or ContractedUnitPrice == 0, EffectiveCost, + // Fall back to the original value for any unhandled scenarios + ContractedCost + ) + | extend ListCost = case( + // If not set or there's no cost, keep the original value + (isnotempty(ListCost) and ListCost != 0) or (EffectiveCost == 0 and BilledCost == 0), ListCost, + // ListCost is 0 in all other scenarios... + // If 0 and there's a contracted cost and prices are the same, use ContractedCost + isnotempty(ContractedCost) and ContractedCost != 0 and ListUnitPrice == ContractedUnitPrice, ContractedCost, + // If 0 and there's a price, calculate the cost based on the price + isnotempty(ListUnitPrice) and ListUnitPrice != 0, ListUnitPrice * PricingQuantity, + // If 0 and there's no price, assume ContractedCost + isempty(ListUnitPrice) or ListUnitPrice == 0, ContractedCost, + // Fall back to the original value for any unhandled scenarios + ListCost + ) + // Merge the rest of the unmodified cost records and remove excess columns + | union (allCosts | where not(tmp_MissingPrices)) + | project-away x_PricingBlockSize1, PricingUnit1, ListUnitPrice1, ContractedUnitPrice1, tmp_MissingPrices, tmp_ReservationPriceLookupKey, tmp_ReservationPriceLookupKey1 + // + | extend SkuPriceDetails = parse_json(SkuPriceDetails) + | extend Tags = parse_json(Tags) + | extend x_SkuDetails = parse_json(x_SkuDetails) + // + // Handle FOCUS 1.0-preview + | extend old_ChargeSubcategory = ChargeSubcategory + | extend old_ChargeCategory = ChargeCategory, ChargeCategory = case( + // Handle FOCUS 1.0-preview ChargeSubcategory + ChargeSubcategory == 'Credit', 'Credit', + ChargeSubcategory == 'Refund', 'Purchase', // We are assuming purchase refunds since we don't have data to indicate usage refunds + ChargeCategory + ) + | extend old_ChargeClass = ChargeClass, ChargeClass = case(ChargeSubcategory == 'Refund', 'Correction', ChargeClass) + // + // Populate CapacityReservationId when not specified + | extend CapacityReservationId = coalesce(CapacityReservationId, tostring(coalesce(x_SkuDetails.VMCapacityReservationId, SkuPriceDetails.VMCapacityReservationId, SkuPriceDetails.x_VMCapacityReservationId))) + | extend old_CapacityReservationStatus = CapacityReservationStatus, CapacityReservationStatus = case( + isempty(CapacityReservationId), '', + isnotempty(CapacityReservationStatus), CapacityReservationStatus, + tolower(x_ResourceType) == 'microsoft.compute/capacityreservationgroups/capacityreservations', 'Unused', + 'Used' + ) + // + // BUG: ChargeFrequency shows "Usage-Based" for monthly recurring savings plan purchases + | extend old_ChargeFrequency = ChargeFrequency, ChargeFrequency = iff(ChargeCategory == 'Purchase' and isnotempty(CommitmentDiscountId) and ChargeFrequency == 'Usage-Based' and ProviderName == 'Microsoft' and x_SourceVersion startswith '1.0', 'Recurring', ChargeFrequency) + // + // Commitment discounts + | extend x_CommitmentDiscountNormalizedRatio = case( + // Calculate from CommitmentDiscountQuantity, if specified + isnotempty(CommitmentDiscountQuantity) and CommitmentDiscountQuantity != 0, CommitmentDiscountQuantity / PricingQuantity / coalesce(x_PricingBlockSize, real(1)), + // Not applicable + isempty(CommitmentDiscountStatus), real(null), + // Parse from SKU details if not specified explicitly + toreal(coalesce(x_SkuDetails.RINormalizationRatio, SkuPriceDetails.RINormalizationRatio, SkuPriceDetails.x_RINormalizationRatio, dynamic(1))) + ) + | extend old_CommitmentDiscountQuantity = CommitmentDiscountQuantity, CommitmentDiscountQuantity = case( + // FOCUS 1.3 + isnotempty(CommitmentDiscountQuantity), CommitmentDiscountQuantity, + // FOCUS 1.0-preview, 1.0 + isempty(CommitmentDiscountStatus), real(null), + CommitmentDiscountCategory == 'Spend', EffectiveCost / coalesce(x_BillingExchangeRate, real(1)), + CommitmentDiscountCategory == 'Usage' and isnotempty(x_CommitmentDiscountNormalizedRatio), PricingQuantity / coalesce(x_PricingBlockSize, real(1)) * x_CommitmentDiscountNormalizedRatio, + real(null) + ) + | extend old_CommitmentDiscountUnit = CommitmentDiscountUnit, CommitmentDiscountUnit = case( + // FOCUS 1.3 + isnotempty(CommitmentDiscountUnit), CommitmentDiscountUnit, + // FOCUS 1.0 + isempty(CommitmentDiscountQuantity), '', + CommitmentDiscountCategory == 'Spend', PricingCurrency, + CommitmentDiscountCategory == 'Usage' and x_CommitmentDiscountNormalizedRatio == real(1), ConsumedUnit, + CommitmentDiscountCategory == 'Usage' and x_CommitmentDiscountNormalizedRatio > real(1), strcat('Normalized ', ConsumedUnit), + '' + ) + | extend old_CommitmentDiscountStatus = CommitmentDiscountStatus, CommitmentDiscountStatus = case( + // FOCUS 1.0+ + isnotempty(CommitmentDiscountStatus), CommitmentDiscountStatus, + // FOCUS 1.0-preview + ChargeSubcategory == 'Used Commitment', 'Used', + ChargeSubcategory == 'Unused Commitment', 'Unused', + '' + ) + | extend x_CommitmentDiscountUtilizationPotential = case( + ChargeCategory == 'Purchase', real(0), + ProviderName == 'Microsoft' and isnotempty(CommitmentDiscountCategory), EffectiveCost, + CommitmentDiscountCategory == 'Usage', ConsumedQuantity, + CommitmentDiscountCategory == 'Spend', EffectiveCost, + real(0) + ) + | extend x_CommitmentDiscountUtilizationAmount = iff(CommitmentDiscountStatus == 'Used', x_CommitmentDiscountUtilizationPotential, real(0)) + // + // Pricing + | extend old_x_AmortizationClass = x_AmortizationClass, x_AmortizationClass = case( + // FOCUS 1.3 + isnotempty(x_AmortizationClass), x_AmortizationClass, + // FOCUS 1.0-preview+ + ChargeCategory == 'Purchase' and (tolower(ResourceId) contains '/microsoft.capacity/reservationorders/' or tolower(ResourceId) contains '/microsoft.billingbenefits/savingsplanorders/'), 'Principal', + ChargeCategory == 'Usage' and isnotempty(CommitmentDiscountId) and isnotempty(CommitmentDiscountStatus), 'Amortized Charge', + '' + ) + | extend old_PricingCategory = PricingCategory, PricingCategory = case( + // FOCUS 1.0+ + isnotempty(PricingCategory), PricingCategory, + // FOCUS 1.0-preview + PricingCategory == 'On-Demand', 'Standard', + PricingCategory == 'Commitment-Based', 'Committed', + '' + ) + // + // Commitment discount utilization + | extend x_CommitmentDiscountUtilizationPotential = case( + ChargeCategory == 'Purchase', real(0), + ProviderName == 'Microsoft' and isnotempty(CommitmentDiscountCategory), EffectiveCost, + CommitmentDiscountCategory == 'Usage', ConsumedQuantity, + CommitmentDiscountCategory == 'Spend', EffectiveCost, + real(0) + ) + | extend x_CommitmentDiscountUtilizationAmount = iff(CommitmentDiscountStatus == 'Used', x_CommitmentDiscountUtilizationPotential, real(0)) + // + // BUG: Fix ContractedCost that has bad values + | extend ContractedCost = iff(ProviderName == 'Microsoft' and isnotempty(PricingQuantity) and isnotempty(x_PricingBlockSize) and ContractedCost != ContractedUnitPrice * PricingQuantity, ContractedUnitPrice * PricingQuantity, ContractedCost) + // + // Handle FOCUS 1.0-preview UsageQuantity/Unit + | extend ConsumedQuantity = iff(ChargeCategory == 'Usage', coalesce(ConsumedQuantity, UsageQuantity, UsageAmount), real(null)) + | extend old_ConsumedUnit = ConsumedUnit, ConsumedUnit = iff(ChargeCategory == 'Usage' and isnotempty(ConsumedQuantity), coalesce(ConsumedUnit, UsageUnit, 'Units'), '') + // + // Convert IDs to lowercase for consistency + | extend BillingAccountId = tolower(BillingAccountId) + | extend CommitmentDiscountId = tolower(CommitmentDiscountId) + // + // BUG: Remove EffectiveCost for commitment discount purchases + | extend old_EffectiveCost = EffectiveCost, EffectiveCost = iff(ChargeCategory == 'Purchase' and isnotempty(CommitmentDiscountId), real(0), EffectiveCost) + | extend old_x_EffectiveCostInUsd = x_EffectiveCostInUsd, x_EffectiveCostInUsd = iff(ChargeCategory == 'Purchase' and isnotempty(CommitmentDiscountId), real(0), x_EffectiveCostInUsd) + // + // Clean up resource columns + | extend old_ResourceId = ResourceId, ResourceId = case( + isnotempty(ResourceId), ResourceId, + ChargeCategory == 'Purchase' and isnotempty(CommitmentDiscountId), CommitmentDiscountId, + ResourceId + ) + | extend old_ResourceName = ResourceName, ResourceName = tolower(case( + isnotempty(ResourceName), ResourceName, + ChargeCategory == 'Purchase' and isnotempty(CommitmentDiscountName), CommitmentDiscountName, + isnotempty(ResourceId), parse_resourceid(ResourceId).ResourceName, + ResourceName + )) + | extend old_x_ResourceType = x_ResourceType, x_ResourceType = case( + isnotempty(x_ResourceType), x_ResourceType, + isnotempty(ResourceId), parse_resourceid(ResourceId).x_ResourceType, + x_ResourceType + ) + | extend old_ResourceType = ResourceType, ResourceType = case( + // Use existing resource type display name unless it's an internal resource type ID + isnotempty(ResourceType) and tolower(ResourceType) != tolower(x_ResourceType) and ResourceType !contains '/', ResourceType, + // Use CommitmentDiscountType for commitment discount purchases + ChargeCategory == 'Purchase' and isnotempty(CommitmentDiscountType), CommitmentDiscountType, + // Look up display name from internal type + isnotempty(x_ResourceType), coalesce(tostring(resource_type(x_ResourceType).SingularDisplayName), ResourceType, x_ResourceType), + ResourceType + ) + // + // Handle missing values + | extend old_PublisherName = PublisherName, PublisherName = case(PublisherName == 'Microsoft Corporation', 'Microsoft', isnotempty(PublisherName), PublisherName, x_PublisherCategory == 'Cloud Provider', ProviderName, '') + // + // Handle FOCUS 1.0-preview Region column + | extend old_Region = Region + | extend old_RegionId = RegionId, RegionId = coalesce(RegionId, iff(ProviderName == 'Microsoft', replace_string(tolower(Region), ' ', ''), Region)) + | extend RegionName = coalesce(RegionName, Region) + // + // SKU properties + | extend x_SkuCoreCount = toint(coalesce(SkuPriceDetails.CoreCount, SkuPriceDetails.x_VCPUs, x_SkuDetails.VCPUs, SkuPriceDetails.x_VCores, x_SkuDetails.VCores, SkuPriceDetails.x_vCores, x_SkuDetails.vCores)) + | extend x_SkuInstanceType = tostring(coalesce(SkuPriceDetails.InstanceType, SkuPriceDetails.x_ServiceType, x_SkuDetails.ServiceType, SkuPriceDetails.x_ServerSku, x_SkuDetails.ServerSku)) + | extend x_SkuOperatingSystem = case( + isnotempty(SkuPriceDetails.OperatingSystem), SkuPriceDetails.OperatingSystem, + coalesce(SkuPriceDetails.x_ImageType, x_SkuDetails.ImageType) == 'Canonical', 'Linux', + coalesce(SkuPriceDetails.x_ImageType, x_SkuDetails.ImageType) == 'Windows Server BYOL', 'Windows Server', + x_SkuMeterSubcategory endswith ' Series Windows', 'Windows Server', + coalesce(SkuPriceDetails.x_ImageType, x_SkuDetails.ImageType) + ) + | extend x_ConsumedCoreHours = iff(ConsumedUnit == 'Hours' and isnotempty(x_SkuCoreCount), x_SkuCoreCount * ConsumedQuantity, real(null)) + | extend SkuPriceDetails = case( + // FOCUS 1.3 + isnotempty(SkuPriceDetails), SkuPriceDetails, + // FOCUS 1.0-preview, 1.0 + parse_json(replace_regex(replace_regex(replace_regex(replace_regex(replace_regex(replace_regex(tostring(x_SkuDetails) + // Prefix all keys with x_ first to avoid double-prefixing + , @'([\{,])"', @'\1"x_') + // CoreCount for number of CPUs/vCPUs/cores/vCores + , @'"x_(VCPUs|VCores|vCores)":', @'"CoreCount":') + // TODO: DiskMaxIops for disk I/O operations per second (IOPS) + // TODO: DiskSpace for disk size in GiB + // TODO: DiskType for the kind of disk (e.g., SSD, HDD, NVMe) + // TODO: GpuCount for the number of GPUs + // InstanceType for the resource size/SKU (e.g., ArmSkuName) + , @'"x_(ServerSku|ServiceType)":', @'"InstanceType":') + // TODO: InstanceSeries for the size family/series + // TODO: MemorySize for the RAM in GiB + // TODO: NetworkMaxIops for network I/O operations per second (IOPS) + // TODO: NetworkMaxThroughput for network max throughput for data transfer in Mbps + // OperatingSystem for the OS name + , @'("x_ImageType":"Canonical")', @'\1,"OperatingSystem":"Linux"') + , @'("x_ImageType":"Windows Server( BYOL)?")', @'\1,"OperatingSystem":"Windows Server"') + , @'("x_ImageType":("[^"]+"))', @'\1,"OperatingSystem":\2') + // TODO: Redundancy for the level of redundancy (e.g., Local, Zonal, Global) + // TODO: StorageClass for the tier of storage (e.g., Hot, Archive, Nearline) + ) + ) + | extend SkuPriceDetails = iff(isempty(SkuPriceDetails.OperatingSystem) and isnotempty(x_SkuOperatingSystem), + parse_json(replace_string(tostring(SkuPriceDetails), '}', strcat(@',"OperatingSystem":"', x_SkuOperatingSystem, '"}'))), + SkuPriceDetails) + // + // Azure Hybrid Benefit + | extend tmp_SqlAhb = tolower(coalesce(x_SkuDetails.AHB, SkuPriceDetails.x_AHB)) + | extend x_SkuLicenseType = case( + ChargeCategory != 'Usage', '', + x_SkuMeterCategory in ('Virtual Machines', 'Virtual Machine Licenses') and (x_SkuMeterSubcategory contains 'Windows' or coalesce(SkuPriceDetails.x_ImageType, x_SkuDetails.ImageType) == 'Windows Server BYOL'), 'Windows Server', + isnotempty(tmp_SqlAhb) or x_SkuMeterSubcategory == 'SQL Server Azure Hybrid Benefit', 'SQL Server', + '' + ) + | extend x_SkuLicenseStatus = case( + isempty(x_SkuLicenseType), '', + coalesce(SkuPriceDetails.x_ImageType, x_SkuDetails.ImageType) == 'Windows Server BYOL' or tmp_SqlAhb == 'true' or x_SkuMeterSubcategory contains 'Azure Hybrid Benefit', 'Enabled', + (x_SkuMeterSubcategory contains 'Windows') or tmp_SqlAhb == 'false', 'Not Enabled', + '' + ) + | extend x_SkuLicenseQuantity = case( + isempty(x_SkuCoreCount) or isempty(x_SkuLicenseType), int(null), + x_SkuCoreCount <= 8, int(8), + x_SkuCoreCount > 8, x_SkuCoreCount, + int(null) + ) + | extend x_SkuLicenseUnit = iff(isnotempty(x_SkuLicenseQuantity), 'Cores', '') + // + // Savings + | extend x_CommitmentDiscountSavings = iff(isempty(ContractedCost) or ContractedCost == 0 or ContractedCost - EffectiveCost < 0.0001, real(0), ContractedCost - EffectiveCost) + | extend x_NegotiatedDiscountSavings = iff(isempty(ListCost) or ListCost == 0 or ListCost - ContractedCost < 0.0001, real(0), ListCost - ContractedCost) + | extend x_TotalSavings = iff(isempty(ListCost) or ListCost == 0 or ListCost - EffectiveCost < 0.0001, real(0), ListCost - EffectiveCost) + | extend x_CommitmentDiscountPercent = iff(isempty(ContractedUnitPrice) or ContractedUnitPrice == 0 or ContractedUnitPrice - x_EffectiveUnitPrice < 0.0001, real(0), (ContractedUnitPrice - x_EffectiveUnitPrice) / ContractedUnitPrice) + | extend x_NegotiatedDiscountPercent = iff(isempty(ListUnitPrice) or ListUnitPrice == 0 or ListUnitPrice - ContractedUnitPrice < 0.0001, real(0), (ListUnitPrice - ContractedUnitPrice) / ListUnitPrice) + | extend x_TotalDiscountPercent = iff(isempty(ListUnitPrice) or ListUnitPrice == 0 or ListUnitPrice - x_EffectiveUnitPrice < 0.0001, real(0), (ListUnitPrice - x_EffectiveUnitPrice) / ListUnitPrice) + // + // Minor fixes + | extend old_BillingPeriodEnd = BillingPeriodEnd, BillingPeriodEnd = startofmonth(BillingPeriodEnd) + | extend old_BillingPeriodStart = BillingPeriodStart, BillingPeriodStart = startofmonth(BillingPeriodStart) + // + // Sort columns and apply final transforms + | project + AllocatedMethodDetails = parse_json(AllocatedMethodDetails), // FOCUS 1.3+ + AllocatedMethodId, // FOCUS 1.3+ + AllocatedResourceId, // FOCUS 1.3+ + AllocatedResourceName, // FOCUS 1.3+ + AllocatedTags = parse_json(AllocatedTags), // FOCUS 1.3+ + AvailabilityZone, + BilledCost, + BillingAccountId, + BillingAccountName, + BillingAccountType, + BillingCurrency, + BillingPeriodEnd, + BillingPeriodStart, + CapacityReservationId, + CapacityReservationStatus, + ChargeCategory, + ChargeClass, + ChargeDescription, + ChargeFrequency, + ChargePeriodEnd, + ChargePeriodStart, + CommitmentDiscountCategory, + CommitmentDiscountId, + CommitmentDiscountName, + CommitmentDiscountQuantity, + CommitmentDiscountStatus, + CommitmentDiscountType, + CommitmentDiscountUnit, + ConsumedQuantity, + ConsumedUnit, + ContractApplied = parse_json(ContractApplied), // FOCUS 1.3+ + ContractedCost = coalesce(ContractedCost, x_OnDemandCost, x_Cost), + ContractedUnitPrice = coalesce(ContractedUnitPrice, x_OnDemandUnitPrice), + EffectiveCost, + HostProviderName = iff(isempty(HostProviderName), PublisherName, HostProviderName), // FOCUS 1.3+; falls back to deprecated PublisherName for back compat + InvoiceId = coalesce(InvoiceId, x_InvoiceId), + InvoiceIssuerName, + ListCost, + ListUnitPrice, + PricingCategory, + PricingCurrency = coalesce(PricingCurrency, x_PricingCurrency), + PricingQuantity, + PricingUnit, + ProviderName, + PublisherName, + RegionId, + RegionName, + ResourceId, + ResourceName, + ResourceType, + ServiceCategory, + ServiceName, + ServiceProviderName = iff(isempty(ServiceProviderName), ProviderName, ServiceProviderName), // FOCUS 1.3+; falls back to deprecated ProviderName for back compat + ServiceSubcategory, // TODO: Populate ServiceSubcategory from ServiceName when missing + SkuId, + SkuMeter = coalesce(SkuMeter, x_SkuMeterName), + SkuPriceDetails, + SkuPriceId, + SubAccountId, + SubAccountName = iff(isempty(SubAccountId), '', SubAccountName), + SubAccountType, + Tags, + x_AccountId = iff(x_AccountId == '-2', '', x_AccountId), + x_AccountName = iff(x_AccountId == '-2', '', x_AccountName), + x_AccountOwnerId = iff(x_AccountId == '-2', '', x_AccountOwnerId), + x_AmortizationClass, + x_BilledCostInUsd, + x_BilledUnitPrice, + x_BillingAccountAgreement = case( + ProviderName == 'Microsoft' and x_BillingAccountId == x_BillingProfileId, 'EA', + ProviderName == 'Microsoft' and x_BillingAccountId != x_BillingProfileId, 'MCA', + ProviderName + ), + x_BillingAccountId, + x_BillingAccountName, + x_BillingExchangeRate, + x_BillingExchangeRateDate, + x_BillingItemCode, + x_BillingItemName, + x_BillingProfileId, + x_BillingProfileName, + x_ChargeId, + x_CommitmentDiscountNormalizedRatio, + x_CommitmentDiscountPercent, + x_CommitmentDiscountSavings, + x_CommitmentDiscountSpendEligibility = '', // TODO: Add x_CommitmentDiscountSpendEligibility for Costs + x_CommitmentDiscountUsageEligibility = '', // TODO: Add x_CommitmentDiscountUsageEligibility for Costs + x_CommitmentDiscountUtilizationAmount, + x_CommitmentDiscountUtilizationPotential, + x_CommodityCode, + x_CommodityName, + x_ComponentName, + x_ComponentType, + x_ConsumedCoreHours, + x_ContractedCostInUsd = coalesce(x_ContractedCostInUsd, x_OnDemandCostInUsd), + x_CostAllocationRuleName, + x_CostCategories = parse_json(x_CostCategories), + x_CostCenter, + x_CostType, + x_Credits = parse_json(x_Credits), + x_CurrencyConversionRate, + x_CustomerId, + x_CustomerName, + x_Discount = parse_json(x_Discount), + x_EffectiveCostInUsd, + x_EffectiveUnitPrice, + x_ExportTime, + x_IngestionTime, + x_InstanceID, + x_InvoiceIssuerId, + x_InvoiceSectionId = case( + x_InvoiceSectionId == '-2', '', + x_InvoiceSectionId + ), + x_InvoiceSectionName = case( + x_InvoiceSectionName == 'Unassigned', '', + x_InvoiceSectionName + ), + x_ListCostInUsd, + x_Location, + x_NegotiatedDiscountPercent, + x_NegotiatedDiscountSavings, + x_Operation, + x_OwnerAccountID, + x_PartnerCreditApplied, + x_PartnerCreditRate, + x_PricingBlockSize, + x_PricingSubcategory, + x_PricingUnitDescription = iff(x_PricingUnitDescription == 'Unassigned', '', x_PricingUnitDescription), + x_Project, + x_PublisherCategory, + x_PublisherId, + x_ResellerId, + x_ResellerName, + x_ResourceGroupName = tolower(x_ResourceGroupName), + x_ResourceType, + x_ServiceCode, + x_ServiceId, + x_ServiceModel, // TODO: Populate from ServiceName when missing + x_ServicePeriodEnd, + x_ServicePeriodStart, + x_SkuCoreCount, + x_SkuDescription, + x_SkuDetails, + x_SkuInstanceType, + x_SkuIsCreditEligible, + x_SkuLicenseQuantity, + x_SkuLicenseStatus, + x_SkuLicenseType, + x_SkuLicenseUnit, + x_SkuMeterCategory, + x_SkuMeterId, + x_SkuMeterSubcategory, + x_SkuOfferId, + x_SkuOperatingSystem, + x_SkuOrderId, + x_SkuOrderName, + x_SkuPartNumber, + x_SkuPlanName, + x_SkuRegion, + x_SkuServiceFamily, + x_SkuTerm, + x_SkuTier, + x_SourceChanges, + x_SourceName, + x_SourceProvider, + x_SourceType, + x_SourceValues = bag_merge( + checkString('BillingPeriodEnd', old_BillingPeriodEnd, BillingPeriodEnd), + checkString('BillingPeriodStart', old_BillingPeriodStart, BillingPeriodStart), + checkString('CapacityReservationStatus', old_CapacityReservationStatus, CapacityReservationStatus), + checkString('ChargeCategory', old_ChargeCategory, ChargeCategory), + checkString('ChargeClass', old_ChargeClass, ChargeClass), + checkString('ChargeSubcategory', old_ChargeSubcategory, ''), // Not included in final schema; use empty string + checkString('ChargeFrequency', old_ChargeFrequency, ChargeFrequency), + checkReal('CommitmentDiscountQuantity', old_CommitmentDiscountQuantity, CommitmentDiscountQuantity), + checkString('CommitmentDiscountUnit', old_CommitmentDiscountUnit, CommitmentDiscountUnit), + checkString('CommitmentDiscountStatus', old_CommitmentDiscountStatus, CommitmentDiscountStatus), + checkReal('ConsumedQuantity', old_ConsumedQuantity, ConsumedQuantity), + checkString('ConsumedUnit', old_ConsumedUnit, ConsumedUnit), + checkReal('ContractedCost', old_ContractedCost, ContractedCost), + checkReal('ContractedUnitPrice', old_ContractedUnitPrice, ContractedUnitPrice), + checkReal('EffectiveCost', old_EffectiveCost, EffectiveCost), + checkReal('ListCost', old_ListCost, ListCost), + checkReal('ListUnitPrice', old_ListUnitPrice, ListUnitPrice), + checkString('PricingCategory', old_PricingCategory, PricingCategory), + checkReal('PricingQuantity', old_PricingQuantity, PricingQuantity), + checkString('ProviderName', old_ProviderName, ProviderName), + checkString('PublisherName', old_PublisherName, PublisherName), + checkString('Region', old_Region, ''), // Not included in final schema; use empty string + checkString('RegionId', old_RegionId, RegionId), + checkString('ResourceId', old_ResourceId, ResourceId), + checkString('ResourceName', old_ResourceName, ResourceName), + checkString('ResourceType', old_ResourceType, ResourceType), + checkString('x_AmortizationClass', old_x_AmortizationClass, x_AmortizationClass), + checkReal('x_EffectiveCostInUsd', old_x_EffectiveCostInUsd, x_EffectiveCostInUsd), + checkReal('x_EffectiveUnitPrice', old_x_EffectiveUnitPrice, x_EffectiveUnitPrice), + checkString('x_ResourceType', old_x_ResourceType, x_ResourceType) + ), + x_SourceVersion, + x_SubproductName, + x_TotalDiscountPercent, + x_TotalSavings, + x_UsageType +} + +// Costs_final_v1_3 table +.create-merge table Costs_final_v1_3 ( + AllocatedMethodDetails: dynamic, // FOCUS 1.3+ + AllocatedMethodId: string, // FOCUS 1.3+ + AllocatedResourceId: string, // FOCUS 1.3+ + AllocatedResourceName: string, // FOCUS 1.3+ + AllocatedTags: dynamic, // FOCUS 1.3+ + AvailabilityZone: string, + BilledCost: real, + BillingAccountId: string, + BillingAccountName: string, + BillingAccountType: string, + BillingCurrency: string, + BillingPeriodEnd: datetime, + BillingPeriodStart: datetime, + CapacityReservationId: string, + CapacityReservationStatus: string, + ChargeCategory: string, + ChargeClass: string, + ChargeDescription: string, + ChargeFrequency: string, + ChargePeriodEnd: datetime, + ChargePeriodStart: datetime, + CommitmentDiscountCategory: string, + CommitmentDiscountId: string, + CommitmentDiscountName: string, + CommitmentDiscountQuantity: real, + CommitmentDiscountStatus: string, + CommitmentDiscountType: string, + CommitmentDiscountUnit: string, + ConsumedQuantity: real, + ConsumedUnit: string, + ContractApplied: dynamic, // FOCUS 1.3+ + ContractedCost: real, + ContractedUnitPrice: real, + EffectiveCost: real, + HostProviderName: string, // FOCUS 1.3+ + InvoiceId: string, + InvoiceIssuerName: string, + ListCost: real, + ListUnitPrice: real, + PricingCategory: string, + PricingCurrency: string, + PricingQuantity: real, + PricingUnit: string, + ProviderName: string, + PublisherName: string, + RegionId: string, + RegionName: string, + ResourceId: string, + ResourceName: string, + ResourceType: string, + ServiceCategory: string, + ServiceName: string, + ServiceProviderName: string, // FOCUS 1.3+ + ServiceSubcategory: string, + SkuId: string, + SkuMeter: string, + SkuPriceDetails: dynamic, + SkuPriceId: string, + SubAccountId: string, + SubAccountName: string, + SubAccountType: string, + Tags: dynamic, + x_AccountId: string, // Azure 1.0-preview(v1)+ + x_AccountName: string, // Azure 1.0-preview(v1)+ + x_AccountOwnerId: string, // Azure 1.0-preview(v1)+ + x_AmortizationClass: string, // Azure 1.2-preview+ + x_BilledCostInUsd: real, // Azure 1.0-preview(v1)+ + x_BilledUnitPrice: real, // Azure 1.0-preview(v1)+ + x_BillingAccountAgreement: string, // Hubs add-on + x_BillingAccountId: string, // Azure 1.0-preview(v1)+ + x_BillingAccountName: string, // Azure 1.0-preview(v1)+ + x_BillingExchangeRate: real, // Azure 1.0-preview(v1)+ + x_BillingExchangeRateDate: datetime, // Azure 1.0-preview(v1)+ + x_BillingItemCode: string, // Alibaba 1.0 + x_BillingItemName: string, // Alibaba 1.0 + x_BillingProfileId: string, // Azure 1.0-preview(v1)+ + x_BillingProfileName: string, // Azure 1.0-preview(v1)+ + x_ChargeId: string, // Azure 1.0-preview(v1) only + x_CommitmentDiscountNormalizedRatio: real, // Azure 1.2-preview+ + x_CommitmentDiscountPercent: real, // Hubs add-on + x_CommitmentDiscountSavings: real, // Hubs add-on + x_CommitmentDiscountSpendEligibility: string, // Hubs add-on + x_CommitmentDiscountUsageEligibility: string, // Hubs add-on + x_CommitmentDiscountUtilizationAmount: real, // Hubs add-on + x_CommitmentDiscountUtilizationPotential: real, // Hubs add-on + x_CommodityCode: string, // Alibaba 1.0 + x_CommodityName: string, // Alibaba 1.0 + x_ComponentName: string, // Tencent 1.0 + x_ComponentType: string, // Tencent 1.0 + x_ConsumedCoreHours: real, // Hubs add-on + x_ContractedCostInUsd: real, // Azure 1.0+ + x_CostAllocationRuleName: string, // Azure 1.0-preview(v1)+ + x_CostCategories: dynamic, // AWS 1.0 (JSON) + x_CostCenter: string, // Azure 1.0-preview(v1)+ + x_CostType: string, // GCP Jan 2024 + x_Credits: dynamic, // GCP Jan 2024 + x_CurrencyConversionRate: real, // GCP Jun 2024 + x_CustomerId: string, // Azure 1.0-preview(v1)+ + x_CustomerName: string, // Azure 1.0-preview(v1)+ + x_Discount: dynamic, // AWS 1.0 (JSON) + x_EffectiveCostInUsd: real, // Azure 1.0-preview(v1)+ + x_EffectiveUnitPrice: real, // Azure 1.0-preview(v1)+ + x_ExportTime: datetime, // GCP Jan 2024 / Tencent 1.0 + x_IngestionTime: datetime, // Hubs add-on + x_InstanceID: string, // Alibaba 1.0 + x_InvoiceIssuerId: string, // Azure 1.0-preview(v1)+ + x_InvoiceSectionId: string, // Azure 1.0-preview(v1)+ + x_InvoiceSectionName: string, // Azure 1.0-preview(v1)+ + x_ListCostInUsd: real, // Azure 1.0-preview(v1)+ + x_Location: string, // GCP Jan 2024 + x_NegotiatedDiscountPercent:real, // Hubs add-on + x_NegotiatedDiscountSavings:real, // Hubs add-on + x_Operation: string, // AWS 1.0 + x_OwnerAccountID: string, // Tencent 1.0 + x_PartnerCreditApplied: string, // Azure 1.0-preview(v1)+ + x_PartnerCreditRate: string, // Azure 1.0-preview(v1)+ + x_PricingBlockSize: real, // Azure 1.0-preview(v1)+ + x_PricingSubcategory: string, // Azure 1.0-preview(v1)+ + x_PricingUnitDescription: string, // Azure 1.0-preview(v1)+ + x_Project: string, // GCP Jan 2024 + x_PublisherCategory: string, // Azure 1.0-preview(v1)+ + x_PublisherId: string, // Azure 1.0-preview(v1)+ + x_ResellerId: string, // Azure 1.0-preview(v1)+ + x_ResellerName: string, // Azure 1.0-preview(v1)+ + x_ResourceGroupName: string, // Azure 1.0-preview(v1)+ + x_ResourceType: string, // Azure 1.0-preview(v1)+ + x_ServiceCode: string, // AWS 1.0 + x_ServiceId: string, // GCP Jan 2024 + x_ServiceModel: string, // Azure 1.2-preview+ + x_ServicePeriodEnd: datetime, // Azure 1.0-preview(v1)+ + x_ServicePeriodStart: datetime, // Azure 1.0-preview(v1)+ + x_SkuCoreCount: int, // Hubs add-on + x_SkuDescription: string, // Azure 1.0-preview(v1)+ + x_SkuDetails: dynamic, // Azure 1.0-preview(v1)+ + x_SkuInstanceType: string, // Hubs add-on + x_SkuIsCreditEligible: bool, // Azure 1.0-preview(v1)+ + x_SkuLicenseQuantity: int, // Hubs add-on + x_SkuLicenseStatus: string, // Hubs add-on + x_SkuLicenseType: string, // Hubs add-on + x_SkuLicenseUnit: string, // Hubs add-on + x_SkuMeterCategory: string, // Azure 1.0-preview(v1)+ + x_SkuMeterId: string, // Azure 1.0-preview(v1)+ + x_SkuMeterSubcategory: string, // Azure 1.0-preview(v1)+ + x_SkuOfferId: string, // Azure 1.0-preview(v1)+ + x_SkuOperatingSystem: string, // Hubs add-on + x_SkuOrderId: string, // Azure 1.0-preview(v1)+ + x_SkuOrderName: string, // Azure 1.0-preview(v1)+ + x_SkuPartNumber: string, // Azure 1.0-preview(v1)+ + x_SkuPlanName: string, // Azure 1.2-preview+ + x_SkuRegion: string, // Azure 1.0-preview(v1)+ + x_SkuServiceFamily: string, // Azure 1.0-preview(v1)+ + x_SkuTerm: int, // Azure 1.0-preview(v1)+ + x_SkuTier: string, // Azure 1.0-preview(v1)+ + x_SourceChanges: string, // Hubs add-on + x_SourceName: string, // Hubs add-on + x_SourceProvider: string, // Hubs add-on + x_SourceType: string, // Hubs add-on + x_SourceValues: dynamic, // Hubs add-on + x_SourceVersion: string, // Hubs add-on + x_SubproductName: string, // Tencent 1.0 + x_TotalDiscountPercent: real, // Hubs add-on + x_TotalSavings: real, // Hubs add-on + x_UsageType: string // AWS 1.0 +) + +// Update policy for Costs_raw -> Costs_final_v1_3 table +.alter table Costs_final_v1_3 policy update +``` +[{ + "IsEnabled": true, + "Source": "Costs_raw", + "Query": "Costs_transform_v1_3()", + "IsTransactional": true, + "PropagateIngestionProperties": true +}] +``` + + +//===| Actual costs |=================================================================================================== +// Supported versions: +// - C360-2025-04 +//====================================================================================================================== + +// ActualCosts_transform_v1_3 function +.create-or-alter function +with (docstring='ActualCost exports transformed to FOCUS 1.3.', folder='Costs') +ActualCosts_transform_v1_3() +{ + // NOTE: All open issues and questions are tracked @ https://github.com/microsoft/finops-toolkit/issues/1111 + ActualCosts_raw + | where ChargeType in ('Purchase', 'Refund') and isnotempty(ReservationId) + // + // + // !!! The rest of the query is reused for both the ActualCosts and AmortizedCosts queries -- Copy all changes in both transform functions !!! + // + // + | extend x_AmortizationClass = case( + ChargeType in ('Purchase', 'Refund') and (tolower(ResourceId) contains '/microsoft.capacity/reservationorders/' or tolower(ResourceId) contains '/microsoft.billingbenefits/savingsplanorders/'), 'Principal', + ChargeType == 'Usage' and isnotempty(ReservationId) and ChargeType != 'UnusedSavingsPlan', 'Amortized Charge', + '' + ) + | extend tmp_ResourceInfo = parse_resourceid(ResourceId) + // TODO: PricingCategory needs to include savings plan usage and spot usage + | extend PricingCategory = case( + x_AmortizationClass == 'Amortized Charge', 'Committed', + ChargeType in ('Usage', 'Purchase'), 'Standard', + '' + ) + | extend x_ResourceType = tostring(tmp_ResourceInfo.x_ResourceType) + | project-rename + PricingQuantity = Quantity, + x_PricingUnitDescription = UnitOfMeasure + | join kind=leftouter (PricingUnits) on x_PricingUnitDescription + | join kind=leftouter (Regions) on ResourceLocation + | join kind=leftouter (ResourceTypes | project x_ResourceType, ResourceType = SingularDisplayName) on x_ResourceType + // TODO: Add the following in 1.2: PublisherName, x_PublisherCategory, x_Environment + | join kind=leftouter (Services | where isnotempty(x_ResourceType) | project x_ResourceType, ServiceName, ServiceCategory, ServiceSubcategory, x_ServiceModel) on x_ResourceType + | join kind=leftouter (Services | where isnotempty(x_ConsumedService) | summarize take_any(ServiceName), take_any(ServiceCategory), take_any(ServiceSubcategory), take_any(x_ServiceModel) by ConsumedService = x_ConsumedService) on ConsumedService + | extend CommitmentDiscountCategory = iff(isnotempty(ReservationId), 'Usage', '') // TODO: CommitmentDiscountCategory needs to handle savings plans + | project + AvailabilityZone = AvailabilityZone, + BilledCost = iff(x_AmortizationClass == 'Amortized Charge', real(0), Cost), + BillingAccountId = strcat('/providers/microsoft.billing/billingaccounts/', BillingAccountId, iff(BillingAccountId == BillingProfileId or isempty(BillingProfileId), '', strcat('/billingprofiles/', BillingProfileId))), + BillingAccountName = coalesce(BillingProfileName, BillingAccountName), + BillingAccountType = iff(BillingAccountId == BillingProfileId or isempty(BillingProfileId), 'Billing Profile', 'Billing Account'), + BillingCurrency, + BillingPeriodEnd = startofmonth(BillingPeriodEndDate, 1), + BillingPeriodStart = startofmonth(BillingPeriodStartDate), + CapacityReservationId = '', + CapacityReservationStatus = '', + ChargeCategory = case( + ChargeType in ('Usage', 'Purchase', 'Credit', 'Tax'), ChargeType, + ChargeType in ('UnusedReservation', 'UnusedSavingsPlan'), 'Usage', + ChargeType == 'Refund', 'Purchase', + 'Adjustment' + ), + ChargeClass = iff(ChargeType == 'Refund', 'Correction', ''), + ChargeDescription = Product, + ChargeFrequency = case( + Frequency == 'UsageBased', 'Usage-Based', + Frequency == 'OneTime', 'One-Time', + Frequency // "Recurring" and any fallback + ), + ChargePeriodEnd = Date + 1d, + ChargePeriodStart = Date, + ChargeSubcategory = '', + // TODO: CommitmentDiscount* columns need to handle savings plans + CommitmentDiscountCategory, + CommitmentDiscountId = iff(isnotempty(ReservationId), strcat('/providers/microsoft.capacity/reservationorders/', ProductOrderId, '/reservations/', ReservationId), ''), + CommitmentDiscountName = ReservationName, + CommitmentDiscountQuantity = real(null), + CommitmentDiscountStatus = case( + isempty(ReservationId), '', + isnotempty(ReservationId) and ChargeType == 'Usage', 'Used', + ChargeType startswith 'Unused', 'Unused', + 'Unused' + ), + CommitmentDiscountType = iff(isnotempty(ReservationId), 'Reservation', ''), + CommitmentDiscountUnit = '', + ConsumedQuantity = PricingQuantity * x_PricingBlockSize, + ConsumedUnit = PricingUnit, + ContractedCost = UnitPrice * PricingQuantity, + ContractedUnitPrice = UnitPrice, + EffectiveCost = iff(x_AmortizationClass == 'Principal', real(0), Cost), + InvoiceId = '', + InvoiceIssuerName = 'Microsoft', + ListCost = real(null), + ListUnitPrice = real(null), + PricingCategory, + PricingCurrency = iff(BillingAccountId == BillingProfileId, BillingCurrency, ''), + PricingQuantity, + PricingUnit, + ProviderName = 'Microsoft', + PublisherName = iff(PublisherType == 'Marketplace', PublisherName, 'Microsoft'), + Region = '', + RegionId, + RegionName, + ResourceId = tostring(tmp_ResourceInfo.ResourceId), + ResourceName = tostring(tmp_ResourceInfo.ResourceName), + ResourceType, + ServiceCategory, + ServiceName, + ServiceSubcategory, + SkuId = '', + SkuMeter = MeterName, + SkuPriceDetails = '', + SkuPriceId = '', + SubAccountId = strcat('/subscriptions/', SubscriptionId), + SubAccountName = iff(isempty(SubscriptionId), '', SubscriptionName), + SubAccountType = iff(isempty(SubscriptionId), '', 'Subscription'), + Tags = iff(Tags startswith '{', Tags, strcat('{', Tags, '}')), + UsageAmount = real(null), + UsageQuantity = real(null), + UsageUnit = '', + x_AccountId = '', + x_AccountName = AccountName, + x_AccountOwnerId = AccountOwnerId, + x_AmortizationClass = '', + x_BilledCostInUsd = real(null), + x_BilledUnitPrice = iff(ChargeType == 'Usage' and isnotempty(ReservationId) and ChargeType != 'UnusedSavingsPlan', real(0), UnitPrice), + x_BillingAccountId = BillingAccountId, + x_BillingAccountName = BillingAccountName, + x_BillingExchangeRate = real(null), + x_BillingExchangeRateDate = datetime(null), + x_BillingItemCode = '', + x_BillingItemName = '', + x_BillingProfileId = BillingProfileId, + x_BillingProfileName = BillingProfileName, + x_ChargeId = '', + x_CommodityCode = '', + x_CommodityName = '', + x_ComponentType = '', + x_ComponentName = '', + x_ContractedCostInUsd = real(null), + x_Cost = real(null), + x_CostAllocationRuleName = '', + x_CostCategories = '', + x_CostCenter = CostCenter, + x_CostType = '', + x_Credits = '', + x_CurrencyConversionRate = real(null), + x_CustomerId = '', + x_CustomerName = '', + x_Discount = '', + x_EffectiveCostInUsd = real(null), + x_EffectiveUnitPrice = iff(ChargeType == 'Usage' and isnotempty(ReservationId) and ChargeType != 'UnusedSavingsPlan', real(0), EffectivePrice), + x_ExportTime = datetime(null), + x_InstanceID = '', + x_InvoiceId = '', + x_InvoiceIssuerId = '', + x_InvoiceSectionId = InvoiceSectionId, + x_InvoiceSectionName = InvoiceSection, + x_ListCostInUsd = real(null), + x_Location = '', + x_OnDemandCost = real(null), + x_OnDemandCostInUsd = real(null), + x_OnDemandUnitPrice = real(null), + x_Operation = '', + x_OwnerAccountID = '', + x_PartnerCreditApplied = '', + x_PartnerCreditRate = '', + x_PricingBlockSize, + x_PricingCurrency = '', + x_PricingSubcategory = case( + // TODO: Add x_SkuTier when supported by C360 -- PricingCategory == 'Standard' and isnotempty(x_SkuTier), 'Tiered', + PricingCategory == 'Standard', 'Standard', + PricingCategory == 'Committed', strcat('Committed ', CommitmentDiscountCategory), + PricingCategory == 'Dynamic', 'Spot', + '' + ), + x_PricingUnitDescription, + x_Project = '', + x_PublisherCategory = iff(PublisherType == 'Marketplace', 'Vendor', 'Cloud Provider'), + x_PublisherId = '', + x_ResellerId = '', + x_ResellerName = '', + x_ResourceGroupName = ResourceGroup, + x_ResourceType, + x_ServiceCode = '', + x_ServiceId = '', + x_ServiceModel, + x_ServicePeriodEnd = datetime(null), + x_ServicePeriodStart = datetime(null), + x_SkuDescription = Product, + x_SkuDetails = iff(AdditionalInfo startswith '{', AdditionalInfo, strcat('{', AdditionalInfo, '}')), + x_SkuIsCreditEligible = IsAzureCreditEligible, + x_SkuMeterCategory = MeterCategory, + x_SkuMeterId = MeterId, + x_SkuMeterName = MeterName, + x_SkuMeterSubcategory = MeterSubCategory, + x_SkuOfferId = OfferId, + x_SkuOrderId = ProductOrderId, + x_SkuOrderName = ProductOrderName, + x_SkuPartNumber = PartNumber, + x_SkuPlanName = '', + x_SkuRegion = MeterRegion, + x_SkuServiceFamily = ServiceFamily, + x_SkuTerm = toint(Term), + x_SkuTier = '', + x_SourceName = 'C360', + x_SourceProvider = 'Microsoft', + x_SourceType = 'ActualCost', + x_SourceVersion = 'C360-2025-04', + x_SubproductName = '', + x_UsageType = '' +} + +// Update policy for ActualCosts_raw -> Costs_raw table +.alter table Costs_raw policy update +``` +[{ + "IsEnabled": true, + "Source": "ActualCosts_raw", + "Query": "ActualCosts_transform_v1_3()", + "IsTransactional": true, + "PropagateIngestionProperties": true +}] +``` + + +//===| Amortized costs |================================================================================================ +// Supported versions: +// - C360-2025-04 +//====================================================================================================================== + +// AmortizedCosts_transform_v1_3 function +.create-or-alter function +with (docstring='ActualCost exports transformed to FOCUS 1.3.', folder='Costs') +AmortizedCosts_transform_v1_3() +{ + // NOTE: All open issues and questions are tracked @ https://github.com/microsoft/finops-toolkit/issues/1111 + AmortizedCosts_raw + // + // + // !!! The rest of the query is reused for both the ActualCosts and AmortizedCosts queries -- Copy all changes in both transform functions !!! + // + // + | extend x_AmortizationClass = case( + ChargeType in ('Purchase', 'Refund') and (tolower(ResourceId) contains '/microsoft.capacity/reservationorders/' or tolower(ResourceId) contains '/microsoft.billingbenefits/savingsplanorders/'), 'Principal', + ChargeType == 'Usage' and isnotempty(ReservationId) and ChargeType != 'UnusedSavingsPlan', 'Amortized Charge', + '' + ) + | extend tmp_ResourceInfo = parse_resourceid(ResourceId) + // TODO: PricingCategory needs to include savings plan usage and spot usage + | extend PricingCategory = case( + x_AmortizationClass == 'Amortized Charge', 'Committed', + ChargeType in ('Usage', 'Purchase'), 'Standard', + '' + ) + | extend x_ResourceType = tostring(tmp_ResourceInfo.x_ResourceType) + | project-rename + PricingQuantity = Quantity, + x_PricingUnitDescription = UnitOfMeasure + | join kind=leftouter (PricingUnits) on x_PricingUnitDescription + | join kind=leftouter (Regions) on ResourceLocation + | join kind=leftouter (ResourceTypes | project x_ResourceType, ResourceType = SingularDisplayName) on x_ResourceType + // TODO: Add the following in 1.2: PublisherName, x_PublisherCategory, x_Environment + | join kind=leftouter (Services | where isnotempty(x_ResourceType) | project x_ResourceType, ServiceName, ServiceCategory, ServiceSubcategory, x_ServiceModel) on x_ResourceType + | join kind=leftouter (Services | where isnotempty(x_ConsumedService) | summarize take_any(ServiceName), take_any(ServiceCategory), take_any(ServiceSubcategory), take_any(x_ServiceModel) by ConsumedService = x_ConsumedService) on ConsumedService + | extend CommitmentDiscountCategory = iff(isnotempty(ReservationId), 'Usage', '') // TODO: CommitmentDiscountCategory needs to handle savings plans + | project + AvailabilityZone = AvailabilityZone, + BilledCost = iff(x_AmortizationClass == 'Amortized Charge', real(0), Cost), + BillingAccountId = strcat('/providers/microsoft.billing/billingaccounts/', BillingAccountId, iff(BillingAccountId == BillingProfileId or isempty(BillingProfileId), '', strcat('/billingprofiles/', BillingProfileId))), + BillingAccountName = coalesce(BillingProfileName, BillingAccountName), + BillingAccountType = iff(BillingAccountId == BillingProfileId or isempty(BillingProfileId), 'Billing Profile', 'Billing Account'), + BillingCurrency, + BillingPeriodEnd = startofmonth(BillingPeriodEndDate, 1), + BillingPeriodStart = startofmonth(BillingPeriodStartDate), + CapacityReservationId = '', + CapacityReservationStatus = '', + ChargeCategory = case( + ChargeType in ('Usage', 'Purchase', 'Credit', 'Tax'), ChargeType, + ChargeType in ('UnusedReservation', 'UnusedSavingsPlan'), 'Usage', + ChargeType == 'Refund', 'Purchase', + 'Adjustment' + ), + ChargeClass = iff(ChargeType == 'Refund', 'Correction', ''), + ChargeDescription = Product, + ChargeFrequency = case( + Frequency == 'UsageBased', 'Usage-Based', + Frequency == 'OneTime', 'One-Time', + Frequency // "Recurring" and any fallback + ), + ChargePeriodEnd = Date + 1d, + ChargePeriodStart = Date, + ChargeSubcategory = '', + // TODO: CommitmentDiscount* columns need to handle savings plans + CommitmentDiscountCategory, + CommitmentDiscountId = iff(isnotempty(ReservationId), strcat('/providers/microsoft.capacity/reservationorders/', ProductOrderId, '/reservations/', ReservationId), ''), + CommitmentDiscountName = ReservationName, + CommitmentDiscountQuantity = real(null), + CommitmentDiscountStatus = case( + isempty(ReservationId), '', + isnotempty(ReservationId) and ChargeType == 'Usage', 'Used', + ChargeType startswith 'Unused', 'Unused', + 'Unused' + ), + CommitmentDiscountType = iff(isnotempty(ReservationId), 'Reservation', ''), + CommitmentDiscountUnit = '', + ConsumedQuantity = PricingQuantity * x_PricingBlockSize, + ConsumedUnit = PricingUnit, + ContractedCost = UnitPrice * PricingQuantity, + ContractedUnitPrice = UnitPrice, + EffectiveCost = iff(x_AmortizationClass == 'Principal', real(0), Cost), + InvoiceId = '', + InvoiceIssuerName = 'Microsoft', + ListCost = real(null), + ListUnitPrice = real(null), + PricingCategory, + PricingCurrency = iff(BillingAccountId == BillingProfileId, BillingCurrency, ''), + PricingQuantity, + PricingUnit, + ProviderName = 'Microsoft', + PublisherName = iff(PublisherType == 'Marketplace', PublisherName, 'Microsoft'), + Region = '', + RegionId, + RegionName, + ResourceId = tostring(tmp_ResourceInfo.ResourceId), + ResourceName = tostring(tmp_ResourceInfo.ResourceName), + ResourceType, + ServiceCategory, + ServiceName, + ServiceSubcategory, + SkuId = '', + SkuMeter = MeterName, + SkuPriceDetails = '', + SkuPriceId = '', + SubAccountId = strcat('/subscriptions/', SubscriptionId), + SubAccountName = iff(isempty(SubscriptionId), '', SubscriptionName), + SubAccountType = iff(isempty(SubscriptionId), '', 'Subscription'), + Tags = iff(Tags startswith '{', Tags, strcat('{', Tags, '}')), + UsageAmount = real(null), + UsageQuantity = real(null), + UsageUnit = '', + x_AccountId = '', + x_AccountName = AccountName, + x_AccountOwnerId = AccountOwnerId, + x_AmortizationClass = '', + x_BilledCostInUsd = real(null), + x_BilledUnitPrice = iff(ChargeType == 'Usage' and isnotempty(ReservationId) and ChargeType != 'UnusedSavingsPlan', real(0), UnitPrice), + x_BillingAccountId = BillingAccountId, + x_BillingAccountName = BillingAccountName, + x_BillingExchangeRate = real(null), + x_BillingExchangeRateDate = datetime(null), + x_BillingItemCode = '', + x_BillingItemName = '', + x_BillingProfileId = BillingProfileId, + x_BillingProfileName = BillingProfileName, + x_ChargeId = '', + x_CommodityCode = '', + x_CommodityName = '', + x_ComponentType = '', + x_ComponentName = '', + x_ContractedCostInUsd = real(null), + x_Cost = real(null), + x_CostAllocationRuleName = '', + x_CostCategories = '', + x_CostCenter = CostCenter, + x_CostType = '', + x_Credits = '', + x_CurrencyConversionRate = real(null), + x_CustomerId = '', + x_CustomerName = '', + x_Discount = '', + x_EffectiveCostInUsd = real(null), + x_EffectiveUnitPrice = iff(ChargeType == 'Usage' and isnotempty(ReservationId) and ChargeType != 'UnusedSavingsPlan', real(0), EffectivePrice), + x_ExportTime = datetime(null), + x_InstanceID = '', + x_InvoiceId = '', + x_InvoiceIssuerId = '', + x_InvoiceSectionId = InvoiceSectionId, + x_InvoiceSectionName = InvoiceSection, + x_ListCostInUsd = real(null), + x_Location = '', + x_OnDemandCost = real(null), + x_OnDemandCostInUsd = real(null), + x_OnDemandUnitPrice = real(null), + x_Operation = '', + x_OwnerAccountID = '', + x_PartnerCreditApplied = '', + x_PartnerCreditRate = '', + x_PricingBlockSize, + x_PricingCurrency = '', + x_PricingSubcategory = case( + // TODO: Add x_SkuTier when supported by C360 -- PricingCategory == 'Standard' and isnotempty(x_SkuTier), 'Tiered', + PricingCategory == 'Standard', 'Standard', + PricingCategory == 'Committed', strcat('Committed ', CommitmentDiscountCategory), + PricingCategory == 'Dynamic', 'Spot', + '' + ), + x_PricingUnitDescription, + x_Project = '', + x_PublisherCategory = iff(PublisherType == 'Marketplace', 'Vendor', 'Cloud Provider'), + x_PublisherId = '', + x_ResellerId = '', + x_ResellerName = '', + x_ResourceGroupName = ResourceGroup, + x_ResourceType, + x_ServiceCode = '', + x_ServiceId = '', + x_ServiceModel, + x_ServicePeriodEnd = datetime(null), + x_ServicePeriodStart = datetime(null), + x_SkuDescription = Product, + x_SkuDetails = iff(AdditionalInfo startswith '{', AdditionalInfo, strcat('{', AdditionalInfo, '}')), + x_SkuIsCreditEligible = IsAzureCreditEligible, + x_SkuMeterCategory = MeterCategory, + x_SkuMeterId = MeterId, + x_SkuMeterName = MeterName, + x_SkuMeterSubcategory = MeterSubCategory, + x_SkuOfferId = OfferId, + x_SkuOrderId = ProductOrderId, + x_SkuOrderName = ProductOrderName, + x_SkuPartNumber = PartNumber, + x_SkuPlanName = '', + x_SkuRegion = MeterRegion, + x_SkuServiceFamily = ServiceFamily, + x_SkuTerm = toint(Term), + x_SkuTier = '', + x_SourceName = 'C360', + x_SourceProvider = 'Microsoft', + x_SourceType = 'ActualCost', + x_SourceVersion = 'C360-2025-04', + x_SubproductName = '', + x_UsageType = '' +} + +// Update policy for AmortizedCosts_raw -> Costs_raw table +.alter table Costs_raw policy update +``` +[{ + "IsEnabled": true, + "Source": "AmortizedCosts_raw", + "Query": "AmortizedCosts_transform_v1_3()", + "IsTransactional": true, + "PropagateIngestionProperties": true +}] +``` + + +//===| CommitmentDiscountUsage |======================================================================================== +// Supported versions: +// - MS EA reservation details: 2023-03-01 -- See https://learn.microsoft.com/azure/cost-management-billing/dataset-schema/reservation-details-ea +// - MS MCA reservation details: 2023-03-01 -- See https://learn.microsoft.com/azure/cost-management-billing/dataset-schema/reservation-details-mca +//====================================================================================================================== + +// CommitmentDiscountUsage_transform_v1_3 function +.create-or-alter function +with (docstring='All commitment discount usage transformed to FOCUS 1.3. This includes reservationdeatils_raw.', folder='Commitment discounts') +CommitmentDiscountUsage_transform_v1_3() +{ + // NOTE: All open issues and questions are tracked @ https://github.com/microsoft/finops-toolkit/issues/1111 + CommitmentDiscountUsage_raw + // + // Set ProviderName + | extend ProviderName = 'Microsoft' + // + // Handle resource columns + | extend ResourceId = tolower(InstanceId) + | extend tmp_ResourceDetails = parse_resourceid(ResourceId) + | extend ResourceName = tostring(tmp_ResourceDetails.ResourceName) + | extend SubAccountId = tostring(tmp_ResourceDetails.SubAccountId) + | extend x_ResourceGroupName = tostring(tmp_ResourceDetails.x_ResourceGroupName) + | extend x_ResourceType = tostring(tmp_ResourceDetails.x_ResourceType) + | lookup kind=leftouter (ResourceTypes | distinct x_ResourceType, ResourceType = SingularDisplayName) on x_ResourceType + | lookup kind=leftouter (Services | distinct x_ResourceType, ServiceName, ServiceCategory, ServiceSubcategory, x_ServiceModel) on x_ResourceType + // + // Sort columns and apply final transforms + | project + ChargePeriodEnd = UsageDate + 1d, + ChargePeriodStart = UsageDate, + CommitmentDiscountCategory = 'Usage', + CommitmentDiscountId = tolower(strcat('/providers/microsoft.capacity/reservationorders/', ReservationOrderId, '/reservations/', ReservationId)), + CommitmentDiscountQuantity = UsedHours * InstanceFlexibilityRatio, + CommitmentDiscountType = 'Reservation', + CommitmentDiscountUnit = case( + InstanceFlexibilityRatio == 1, 'Hours', + InstanceFlexibilityRatio != 1, 'Normalized Hours', + '' + ), + ConsumedQuantity = UsedHours, + ProviderName, + ResourceId, + ResourceName, + ResourceType, + ServiceCategory, + ServiceName, + ServiceSubcategory, + SubAccountId, + x_CommitmentDiscountCommittedCount = TotalReservedQuantity, + x_CommitmentDiscountCommittedAmount = ReservedHours, + // TODO: Is this needed? -- x_CommitmentDiscountKind = Kind, + x_CommitmentDiscountNormalizedGroup = iff(InstanceFlexibilityGroup == 'NA', '', InstanceFlexibilityGroup), + x_CommitmentDiscountNormalizedRatio = InstanceFlexibilityRatio, + x_IngestionTime = ingestion_time(), + x_ResourceGroupName, + x_ResourceType, + // x_RowId = hash_sha256(strcat( + // // DO NOT CHANGE COLUMNS OR COLUMN ORDER + // CommitmentDiscountId, + // ResourceId, + // ChargePeriodStart + // )), + x_ServiceModel, + x_SkuOrderId = ReservationOrderId, + x_SkuSize = iff(SkuName == 'NA', '', SkuName), + x_SourceName = coalesce(x_SourceName, iff(ProviderName == 'Microsoft', 'Cost Management', ProviderName)), + x_SourceProvider = coalesce(x_SourceProvider, ProviderName), + x_SourceType = coalesce(x_SourceType, iff(ProviderName == 'Microsoft', 'ReservationDetails', '')), + x_SourceVersion = coalesce(x_SourceVersion, iff(ProviderName == 'Microsoft', '2024-03-01', '')) +} + +// CommitmentDiscountUsage_final_v1_3 table +.create-merge table CommitmentDiscountUsage_final_v1_3 ( + ChargePeriodEnd: datetime, // Hubs add-on + ChargePeriodStart: datetime, // MS 2023-03-01 + CommitmentDiscountCategory: string, // Hubs add-on + CommitmentDiscountId: string, // MS 2023-03-01 + CommitmentDiscountQuantity: real, // MS 2023-03-01 + CommitmentDiscountType: string, // Hubs add-on + CommitmentDiscountUnit: string, // Hubs add-on + ConsumedQuantity: real, // MS 2023-03-01 + ProviderName: string, // Hubs add-on + ResourceId: string, // MS 2023-03-01 + ResourceName: string, // Hubs add-on + ResourceType: string, // Hubs add-on + ServiceCategory: string, // Hubs add-on + ServiceName: string, // Hubs add-on + ServiceSubcategory: string, // Hubs add-on + SubAccountId: string, // Hubs add-on + x_CommitmentDiscountCommittedCount: real, // MS 2023-03-01 + x_CommitmentDiscountCommittedAmount: real, // MS 2023-03-01 + x_CommitmentDiscountNormalizedGroup: string, // MS 2023-03-01 + x_CommitmentDiscountNormalizedRatio: real, // MS 2023-03-01 + x_IngestionTime: datetime, // Hubs add-on + x_ResourceGroupName: string, // Hubs add-on + x_ResourceType: string, // Hubs add-on + x_ServiceModel: string, // Hubs add-on + x_SkuOrderId: string, // MS 2023-03-01 + x_SkuSize: string, // MS 2023-03-01 + x_SourceName: string, // Hubs add-on + x_SourceProvider: string, // Hubs add-on + x_SourceType: string, // Hubs add-on + x_SourceVersion: string // Hubs add-on +) + +// Update policy for CommitmentDiscountUsage_raw -> CommitmentDiscountUsage_final_v1_3 table +.alter table CommitmentDiscountUsage_final_v1_3 policy update +``` +[{ + "IsEnabled": true, + "Source": "CommitmentDiscountUsage_raw", + "Query": "CommitmentDiscountUsage_transform_v1_3()", + "IsTransactional": true, + "PropagateIngestionProperties": true +}] +``` + + +//===| Recommendations |================================================================================================ +// Supported datasets/versions: +// - MS CM EA reservation recommendations: 2023-05-01 -- See https://learn.microsoft.com/azure/cost-management-billing/dataset-schema/reservation-recommendations-ea +// - MS CM MCA reservation recommendations: 2023-05-01 -- See https://learn.microsoft.com/azure/cost-management-billing/dataset-schema/reservation-recommendations-mca +//====================================================================================================================== + +// Recommendations_transform_v1_3 function +.create-or-alter function +with (docstring='All recommendations transformed to FOCUS 1.3.', folder='Recommendations') +Recommendations_transform_v1_3() +{ + // NOTE: All open issues and questions are tracked @ https://github.com/microsoft/finops-toolkit/issues/1111 + let isoMonths = (duration: string) { + let number = toint(replace_regex(duration, @'[PMY]', '')); + toint(case( + duration == '', int(null), + duration endswith "Y", number * 12, + duration endswith "M", number, + -1 + )) + }; + Recommendations_raw + | extend x_IngestionTime = ingestion_time() + // + // Set ProviderName + | extend ProviderName = 'Microsoft' + // + // Set source columns + | extend x_SourceName = coalesce(x_SourceName, iff(ProviderName == 'Microsoft', 'Cost Management', ProviderName)) + | extend x_SourceProvider = coalesce(x_SourceProvider, ProviderName) + | extend x_SourceType = coalesce(x_SourceType, iff(ProviderName == 'Microsoft', 'ReservationRecommendations', '')) + | extend x_SourceVersion = coalesce(x_SourceVersion, iff(ProviderName == 'Microsoft', '2023-05-01', '')) + // + // Convert JSON cost columns to real + | extend CostWithNoReservedInstances = case(isnotempty(CostWithNoReservedInstances), CostWithNoReservedInstances, isnotempty(CostWithNoReservedInstancesJson), toreal(extract(@'"value":([0-9\.]+)', 1, CostWithNoReservedInstancesJson)), CostWithNoReservedInstances) + | extend NetSavings = case(isnotempty(NetSavings), NetSavings, isnotempty(NetSavingsJson), toreal(extract(@'"value":([0-9\.]+)', 1, NetSavingsJson)), NetSavings) + | extend TotalCostWithReservedInstances = case(isnotempty(TotalCostWithReservedInstances), TotalCostWithReservedInstances, isnotempty(TotalCostWithReservedInstancesJson), toreal(extract(@'"value":([0-9\.]+)', 1, TotalCostWithReservedInstancesJson)), TotalCostWithReservedInstances) + // + // Parse x_RecommendationDetails from JSON string (ARG queries serialize to string for Parquet compatibility) + | extend x_RecommendationDetails = iff(gettype(x_RecommendationDetails) == 'string', parse_json(tostring(x_RecommendationDetails)), x_RecommendationDetails) + // + // Normalize x_RecommendationDetails keys to x_PascalCase (Advisor extendedProperties use camelCase) + // Guard: inject a placeholder key so mv-apply doesn't drop rows with null/empty bags + | extend x_RecommendationDetails = bag_merge(coalesce(x_RecommendationDetails, dynamic({})), bag_pack('__placeholder', '')) + | mv-apply k = bag_keys(x_RecommendationDetails) on ( + where isnotempty(tostring(k)) and tostring(k) != '__placeholder' + | extend newKey = iff(tostring(k) startswith 'x_', tostring(k), strcat('x_', toupper(substring(tostring(k), 0, 1)), substring(tostring(k), 1))) + | summarize x_RecommendationDetails = make_bag(bag_pack(newKey, x_RecommendationDetails[tostring(k)])) + ) + // + // Build recommendation details + | lookup kind=leftouter (Regions | summarize RegionName = make_set(RegionName)[0] by Location = RegionId) on Location + | extend x_RecommendationDetails = case( + // Use incoming x_RecommendationDetails first + isnotempty(x_RecommendationDetails), x_RecommendationDetails, + // Create one for reservation recommendations if needed + x_SourceType == 'ReservationRecommendations', bag_pack( + 'CommitmentDiscountNormalizedGroup', InstanceFlexibilityGroup, + 'CommitmentDiscountNormalizedRatio', InstanceFlexibilityRatio, + 'CommitmentDiscountNormalizedSize', NormalizedSize, + 'CommitmentDiscountResourceType', ResourceType, + 'CommitmentDiscountScope', Scope, + 'LookbackPeriodDuration', case( + LookBackPeriod matches regex @'^Last([0-9]+)Days$', replace_regex(LookBackPeriod, @'^Last([0-9]+)Days$', @'P\1D'), + LookBackPeriod matches regex @'^[0-9]+$', strcat('P', LookBackPeriod, 'D'), + '' + ), + 'LookbackPeriodStart', FirstUsageDate, + 'RecommendedQuantity', RecommendedQuantity, + 'RecommendedQuantityNormalized', RecommendedQuantityNormalized, + 'RegionId', Location, + 'RegionName', RegionName, + 'SkuMeterId', MeterId, + 'SkuPriceDetails', SkuProperties, + 'SkuSize', coalesce(SKU, SkuName), + 'SkuTerm', isoMonths(Term) + ), + dynamic({}) + ) + // + // Prefer specified date, then fall back to generating a date based on reservation recommendation lookback period, then validate to ensure it's not in the future + | extend x_RecommendationDate = coalesce(x_RecommendationDate, FirstUsageDate + (toint(extract(@'^P([0-9]+)D$', 1, tostring(x_RecommendationDetails.LookbackPeriodDuration))) * 1d)) + | extend x_RecommendationDate = iff(x_RecommendationDate > now(), startofday(now()), x_RecommendationDate) + // + // Derive x_ResourceType from ResourceId + | extend tmp_ResourceType = tostring(parse_resourceid(ResourceId).x_ResourceType) + | extend x_RecommendationDetails = iff(isnotempty(tmp_ResourceType), bag_merge(bag_pack('x_ResourceType', tmp_ResourceType), x_RecommendationDetails), x_RecommendationDetails) + // + // Set ResourceType display name from x_ResourceType code + | extend ResourceType = coalesce(ResourceType, tostring(resource_type(tmp_ResourceType).SingularDisplayName), tmp_ResourceType) + // + | project + ProviderName, + ResourceId = tolower(ResourceId), // Force lowercase for consistent grouping/filtering + ResourceName = tolower(iff(tmp_ResourceType =~ 'microsoft.resources/subscriptions', coalesce(SubAccountName, ResourceName), ResourceName)), // Use subscription name for subscription-level recommendations + ResourceType, + SubAccountId = coalesce(SubAccountId, iff(isnotempty(SubscriptionId), strcat('/subscriptions/', SubscriptionId), '')), + SubAccountName, + x_EffectiveCostAfter = coalesce(x_EffectiveCostAfter, TotalCostWithReservedInstances), + x_EffectiveCostBefore = coalesce(x_EffectiveCostBefore, CostWithNoReservedInstances), + x_EffectiveCostSavings = coalesce(x_EffectiveCostSavings, NetSavings, toreal(x_RecommendationDetails.x_SavingsAmount), toreal(x_RecommendationDetails.x_AnnualSavingsAmount) / 12), + x_IngestionTime, + x_RecommendationCategory, // TODO: Set for reservation recommendations + x_RecommendationDate, + x_RecommendationDescription = coalesce(x_RecommendationDescription, tostring(x_RecommendationDetails.x_RecommendationSolution), tostring(x_RecommendationDetails.x_RecommendationSubCategory)), + x_RecommendationDetails, + x_RecommendationId = tolower(x_RecommendationId), // TODO: Set for reservation recommendations; force lowercase for consistent grouping/filtering + x_ResourceGroupName = tolower(x_ResourceGroupName), // Force lowercase for consistent grouping/filtering + x_SourceName, + x_SourceProvider, + x_SourceType, + x_SourceVersion +} + +// Recommendations_final_v1_3 table +.create-merge table Recommendations_final_v1_3 ( + ProviderName: string, + ResourceId: string, + ResourceName: string, + ResourceType: string, + SubAccountId: string, + SubAccountName: string, + x_EffectiveCostAfter: real, + x_EffectiveCostBefore: real, + x_EffectiveCostSavings: real, + x_IngestionTime: datetime, + x_RecommendationCategory: string, + x_RecommendationDate: datetime, + x_RecommendationDescription: string, + x_RecommendationDetails: dynamic, + x_RecommendationId: string, + x_ResourceGroupName: string, + x_SourceName: string, + x_SourceProvider: string, + x_SourceType: string, + x_SourceVersion: string +) + +// Update policy for Recommendations_raw -> Recommendations_final_v1_3 table +.alter table Recommendations_final_v1_3 policy update +``` +[{ + "IsEnabled": true, + "Source": "Recommendations_raw", + "Query": "Recommendations_transform_v1_3()", + "IsTransactional": true, + "PropagateIngestionProperties": true +}] +``` + + +//===| Transactions |=================================================================================================== +// Supported versions: +// - MS CM EA reservation transactions: 2023-05-01 -- See https://learn.microsoft.com/azure/cost-management-billing/dataset-schema/reservation-transactions-ea +// - MS CM MCA reservation transactions: 2023-05-01 -- See https://learn.microsoft.com/azure/cost-management-billing/dataset-schema/reservation-transactions-mca +//====================================================================================================================== + +// Transactions_transform_v1_3 function +.create-or-alter function +with (docstring='All transactions transformed to FOCUS 1.3.', folder='Transactions') +Transactions_transform_v1_3() +{ + // NOTE: All open issues and questions are tracked @ https://github.com/microsoft/finops-toolkit/issues/1111 + let isoMonths = (duration: string) { + let number = toint(replace_regex(duration, @'[PMY]', '')); + toint(case( + duration == '', int(null), + duration endswith "Y", number * 12, + duration endswith "M", number, + -1 + )) + }; + Transactions_raw + // + // Set ProviderName + | extend ProviderName = 'Microsoft' + // + // Set source columns + | extend x_SourceName = coalesce(x_SourceName, iff(ProviderName == 'Microsoft', 'Cost Management', ProviderName)) + | extend x_SourceProvider = coalesce(x_SourceProvider, ProviderName) + | extend x_SourceType = coalesce(x_SourceType, iff(ProviderName == 'Microsoft', 'ReservationTransactions', '')) + | extend x_SourceVersion = coalesce(x_SourceVersion, iff(ProviderName == 'Microsoft', '2023-05-01', '')) + // + // Handle BillingPeriodStart/End + | extend BillingMonth = tostring(BillingMonth) + | extend BillingPeriodStart = iff(isempty(BillingMonth), datetime(null), todatetime(strcat(substring(BillingMonth, 0, 4), "-", substring(BillingMonth, 4, 2), "-", substring(BillingMonth, 6, 2)))) + | extend BillingPeriodEnd = iff(isempty(BillingMonth), datetime(null), startofmonth(endofmonth(BillingPeriodStart) + 1d)) + // + // Sort columns and apply final transforms + | project + BilledCost = Amount, + BillingAccountId = case( + BillingProfileId startswith '/', BillingProfileId, + isnotempty(CurrentEnrollmentId), strcat('/providers/Microsoft.Billing/billingAccounts/', CurrentEnrollmentId), + isnotempty(BillingProfileId), strcat('/providers/Microsoft.Billing/billingProfiles/', BillingProfileId), + '' + ), + BillingAccountName = coalesce(BillingProfileName, CurrentEnrollmentId), + BillingCurrency = Currency, + BillingPeriodEnd, + BillingPeriodStart, + ChargeCategory = case( + EventType in ('Cancel', 'Purchase', 'Refund'), 'Purchase', + 'Adjustment' + ), + ChargeClass = case( + EventType == 'Cancel', 'Cancel', // FOCUS does not handle this scenario + EventType == 'Refund', 'Correction', + '' + ), + ChargeDescription = Description, + ChargeFrequency = case( + BillingFrequency == 'OneTime', 'One-Time', + BillingFrequency == 'Recurring', 'Recurring', + BillingFrequency + ), + ChargePeriodStart = EventDate, + InvoiceId, + PricingQuantity = Quantity, + PricingUnit = 'Reservations', + ProviderName, + RegionId = Region, + RegionName = Region, + SubAccountId = iff(isempty(PurchasingSubscriptionGuid), '', strcat('/subscriptions/', PurchasingSubscriptionGuid)), + SubAccountName = iff(isempty(PurchasingSubscriptionGuid), '', PurchasingSubscriptionName), + x_AccountName = AccountName, + x_AccountOwnerId = AccountOwnerEmail, + x_CostCenter = CostCenter, + x_InvoiceNumber = Invoice, + x_InvoiceSectionId = InvoiceSectionId, + x_InvoiceSectionName = coalesce(InvoiceSectionName, DepartmentName), + x_IngestionTime = ingestion_time(), + x_MonetaryCommitment = MonetaryCommitment, + x_Overage = Overage, + x_PurchasingBillingAccountId = PurchasingEnrollment, + x_SkuOrderId = ReservationOrderId, + x_SkuOrderName = ReservationOrderName, + x_SkuSize = ArmSkuName, + x_SkuTerm = isoMonths(Term), + x_SourceName, + x_SourceProvider, + x_SourceType, + x_SourceVersion, + x_SubscriptionId = PurchasingSubscriptionGuid, + x_TransactionType = EventType +} + +// Transactions_final_v1_3 table +.create-merge table Transactions_final_v1_3 ( + BilledCost: real, // MS CM EA+MCA 2023-05-01 + BillingAccountId: string, // MS CM EA+MCA 2023-05-01 + BillingAccountName: string, // MS CM EA+MCA 2023-05-01 + BillingCurrency: string, // MS CM EA+MCA 2023-05-01 + BillingPeriodEnd: datetime, // MS CM EA+MCA 2023-05-01 + BillingPeriodStart: datetime, // MS CM EA+MCA 2023-05-01 + ChargeCategory: string, // Hubs add-on + ChargeClass: string, // Hubs add-on + ChargeDescription: string, // MS CM EA+MCA 2023-05-01 + ChargeFrequency: string, // MS CM EA+MCA 2023-05-01 + ChargePeriodStart: datetime, // MS CM EA+MCA 2023-05-01 + InvoiceId: string, // MS CM MCA 2023-05-01 + PricingQuantity: real, // MS CM EA+MCA 2023-05-01 + PricingUnit: string, // Hubs add-on + ProviderName: string, // Hubs add-on + RegionId: string, // MS CM EA+MCA 2023-05-01 + RegionName: string, // MS CM EA+MCA 2023-05-01 + SubAccountId: string, // MS CM EA+MCA 2023-05-01 + SubAccountName: string, // MS CM EA+MCA 2023-05-01 + x_AccountName: string, // MS CM EA 2023-05-01 + x_AccountOwnerId: string, // MS CM EA 2023-05-01 + x_CostCenter: string, // MS CM EA 2023-05-01 + x_InvoiceNumber: string, // MS CM MCA 2023-05-01 + x_InvoiceSectionId: string, // MS CM MCA 2023-05-01 + x_InvoiceSectionName: string, // MS CM MCA 2023-05-01 + x_IngestionTime: datetime, // Hubs add-on + x_MonetaryCommitment: real, // MS CM EA 2023-05-01 + x_Overage: real, // MS CM EA 2023-05-01 + x_PurchasingBillingAccountId: string, // MS CM EA 2023-05-01 + x_SkuOrderId: string, // MS CM EA+MCA 2023-05-01 + x_SkuOrderName: string, // MS CM EA+MCA 2023-05-01 + x_SkuSize: string, // MS CM EA+MCA 2023-05-01 + x_SkuTerm: int, // MS CM EA+MCA 2023-05-01 + x_SourceName: string, // Hubs add-on + x_SourceProvider: string, // Hubs add-on + x_SourceType: string, // Hubs add-on + x_SourceVersion: string, // Hubs add-on + x_SubscriptionId: string, // MS CM EA+MCA 2023-05-01 + x_TransactionType: string // MS CM EA+MCA 2023-05-01 +) + +// Update policy for Transactions_raw -> Transactions_final_v1_3 table +.alter table Transactions_final_v1_3 policy update +``` +[{ + "IsEnabled": true, + "Source": "Transactions_raw", + "Query": "Transactions_transform_v1_3()", + "IsTransactional": true, + "PropagateIngestionProperties": true +}] +``` From c56b616547c3f00d912b2c7bcfe2c9b6e60ee555 Mon Sep 17 00:00:00 2001 From: Michael Flanakin Date: Sun, 17 May 2026 13:24:08 -0700 Subject: [PATCH 2/6] Add FOCUS 1.4 ingestion (IngestionSetup_v1_4.kql) Consolidates FOCUS 1.3 and 1.4 column additions into a single v1_4 ingestion setup file. Removes the intermediate v1_3 file, adds ContractCommitment_raw table definition, and updates app.bicep and .build.config to reference v1_4. Co-Authored-By: Claude --- src/templates/finops-hub/.build.config | 2 +- .../Microsoft.FinOpsHubs/Analytics/app.bicep | 2 +- .../scripts/IngestionSetup_RawTables.kql | 96 +++++++++ ...Setup_v1_3.kql => IngestionSetup_v1_4.kql} | 201 +++++++++++++----- 4 files changed, 246 insertions(+), 55 deletions(-) rename src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/{IngestionSetup_v1_3.kql => IngestionSetup_v1_4.kql} (92%) diff --git a/src/templates/finops-hub/.build.config b/src/templates/finops-hub/.build.config index 0963fb494..59ccbdc8c 100644 --- a/src/templates/finops-hub/.build.config +++ b/src/templates/finops-hub/.build.config @@ -34,7 +34,7 @@ "modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_RawTables.kql", "modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_v1_0.kql", "modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_v1_2.kql", - "modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_v1_3.kql" + "modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_v1_4.kql" ] }, { diff --git a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/app.bicep b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/app.bicep index 42cd2738b..e20e982bd 100644 --- a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/app.bicep +++ b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/app.bicep @@ -378,7 +378,7 @@ module ingestion_VersionedScripts '../../fx/hub-database.bicep' = if (useAzure) scripts: { v1_0: loadTextContent('scripts/IngestionSetup_v1_0.kql') v1_2: loadTextContent('scripts/IngestionSetup_v1_2.kql') - v1_3: loadTextContent('scripts/IngestionSetup_v1_3.kql') + v1_4: loadTextContent('scripts/IngestionSetup_v1_4.kql') } continueOnErrors: continueOnErrors forceUpdateTag: forceUpdateTag diff --git a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_RawTables.kql b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_RawTables.kql index 83a3be40e..cdc281659 100644 --- a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_RawTables.kql +++ b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_RawTables.kql @@ -335,6 +335,102 @@ .alter table CommitmentDiscountUsage_raw policy streamingingestion disable +//===| Contract commitment |============================================================================================ +// Supported versions: +// - FOCUS 1.3+ (no Cost Management export shipped yet) +//====================================================================================================================== + +// ContractCommitment_raw table -- Create the table if it doesn't exist +.create-merge table ContractCommitment_raw ( ignore: string ) + +// ContractCommitment_raw table -- Remove all columns to allow changing column types +.alter table ContractCommitment_raw ( ignore: string ) + +// ContractCommitment_raw table -- Redefine all columns +.alter table ContractCommitment_raw ( + BenefitCategory: string, // FOCUS 1.4+ + BillingCurrency: string, // FOCUS 1.3+ + ContractCommitmentApplicability: string, // FOCUS 1.4+ (JSON) + ContractCommitmentCategory: string, // FOCUS 1.3+ + ContractCommitmentCost: real, // FOCUS 1.3+ + ContractCommitmentId: string, // FOCUS 1.3+ + ContractCommitmentPeriodEnd: datetime, // FOCUS 1.3+ + ContractCommitmentPeriodStart: datetime, // FOCUS 1.3+ + ContractCommitmentQuantity: real, // FOCUS 1.3+ + ContractCommitmentType: string, // FOCUS 1.3+ + ContractCommitmentUnit: string, // FOCUS 1.3+ + ContractId: string, // FOCUS 1.3+ + ContractPeriodEnd: datetime, // FOCUS 1.3+ + ContractPeriodStart: datetime, // FOCUS 1.3+ + Created: datetime, // FOCUS 1.4+ + DiscountPercentage: real, // FOCUS 1.4+ + DurationType: string, // FOCUS 1.4+ + FulfillmentInterval: string, // FOCUS 1.4+ + InvoiceIssuerName: string, // FOCUS 1.3+ + LastUpdated: datetime, // FOCUS 1.4+ + LifecycleStatus: string, // FOCUS 1.4+ + Model: string, // FOCUS 1.4+ + OfferCategory: string, // FOCUS 1.4+ + PaymentInterval: string, // FOCUS 1.4+ + PaymentModel: string, // FOCUS 1.4+ + PaymentUpfrontPercentage: real, // FOCUS 1.4+ + PricingCurrency: string, // FOCUS 1.3+ + PricingCurrencyContractCommitmentCost: real, // FOCUS 1.4+ + x_SourceName: string, // Hubs v1_3+ + x_SourceProvider: string, // Hubs v1_3+ + x_SourceType: string, // Hubs v1_3+ + x_SourceVersion: string // Hubs v1_3+ +) + +// ContractCommitment_raw ingestion mapping +.create-or-alter table ContractCommitment_raw ingestion parquet mapping "ContractCommitment_raw_mapping" +``` +[ + { "Column": "BenefitCategory", "Properties": { "Field": "BenefitCategory" } }, + { "Column": "BillingCurrency", "Properties": { "Field": "BillingCurrency" } }, + { "Column": "ContractCommitmentApplicability", "Properties": { "Field": "ContractCommitmentApplicability" } }, + { "Column": "ContractCommitmentCategory", "Properties": { "Field": "ContractCommitmentCategory" } }, + { "Column": "ContractCommitmentCost", "Properties": { "Field": "ContractCommitmentCost" } }, + { "Column": "ContractCommitmentId", "Properties": { "Field": "ContractCommitmentId" } }, + { "Column": "ContractCommitmentPeriodEnd", "Properties": { "Field": "ContractCommitmentPeriodEnd" } }, + { "Column": "ContractCommitmentPeriodStart", "Properties": { "Field": "ContractCommitmentPeriodStart" } }, + { "Column": "ContractCommitmentQuantity", "Properties": { "Field": "ContractCommitmentQuantity" } }, + { "Column": "ContractCommitmentType", "Properties": { "Field": "ContractCommitmentType" } }, + { "Column": "ContractCommitmentUnit", "Properties": { "Field": "ContractCommitmentUnit" } }, + { "Column": "ContractId", "Properties": { "Field": "ContractId" } }, + { "Column": "ContractPeriodEnd", "Properties": { "Field": "ContractPeriodEnd" } }, + { "Column": "ContractPeriodStart", "Properties": { "Field": "ContractPeriodStart" } }, + { "Column": "Created", "Properties": { "Field": "Created" } }, + { "Column": "DiscountPercentage", "Properties": { "Field": "DiscountPercentage" } }, + { "Column": "DurationType", "Properties": { "Field": "DurationType" } }, + { "Column": "FulfillmentInterval", "Properties": { "Field": "FulfillmentInterval" } }, + { "Column": "InvoiceIssuerName", "Properties": { "Field": "InvoiceIssuerName" } }, + { "Column": "LastUpdated", "Properties": { "Field": "LastUpdated" } }, + { "Column": "LifecycleStatus", "Properties": { "Field": "LifecycleStatus" } }, + { "Column": "Model", "Properties": { "Field": "Model" } }, + { "Column": "OfferCategory", "Properties": { "Field": "OfferCategory" } }, + { "Column": "PaymentInterval", "Properties": { "Field": "PaymentInterval" } }, + { "Column": "PaymentModel", "Properties": { "Field": "PaymentModel" } }, + { "Column": "PaymentUpfrontPercentage", "Properties": { "Field": "PaymentUpfrontPercentage" } }, + { "Column": "PricingCurrency", "Properties": { "Field": "PricingCurrency" } }, + { "Column": "PricingCurrencyContractCommitmentCost", "Properties": { "Field": "PricingCurrencyContractCommitmentCost" } }, + { "Column": "x_SourceName", "Properties": { "Field": "x_SourceName" } }, + { "Column": "x_SourceProvider", "Properties": { "Field": "x_SourceProvider" } }, + { "Column": "x_SourceType", "Properties": { "Field": "x_SourceType" } }, + { "Column": "x_SourceVersion", "Properties": { "Field": "x_SourceVersion" } } +] +``` + +// ContractCommitment_raw retention policy (clear historical data) +.alter-merge table ContractCommitment_raw policy retention softdelete = 0d recoverability = disabled + +// ContractCommitment_raw retention policy (set the user-defined retention period) +.alter-merge table ContractCommitment_raw policy retention softdelete = $$rawRetentionInDays$$d recoverability = disabled + +// Disable ContractCommitment_raw streaming ingestion (required for Fabric) +.alter table ContractCommitment_raw policy streamingingestion disable + + //===| Costs |========================================================================================================== // Supported versions: // - MS: 1.0, 1.0-preview(v1) -- See https://aka.ms/costmgmt/exports/focus diff --git a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_v1_3.kql b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_v1_4.kql similarity index 92% rename from src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_v1_3.kql rename to src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_v1_4.kql index 70d29a2ed..84e8d6fcd 100644 --- a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_v1_3.kql +++ b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_v1_4.kql @@ -14,10 +14,10 @@ // - MS MCA 2023-05-01 -- See https://learn.microsoft.com/azure/cost-management-billing/dataset-schema/price-sheet-mca //====================================================================================================================== -// Prices_transform_v1_3 function +// Prices_transform_v1_4 function .create-or-alter function -with (docstring='Transforms Prices_raw into FOCUS 1.3.', folder='Prices') -Prices_transform_v1_3() +with (docstring='Transforms Prices_raw into FOCUS 1.4.', folder='Prices') +Prices_transform_v1_4() { // NOTE: All open issues and questions are tracked @ https://github.com/microsoft/finops-toolkit/issues/1111 let isoMonths = (duration: string) { @@ -196,8 +196,8 @@ Prices_transform_v1_3() x_TotalUnitPriceDiscountPercent = 1.0 * x_TotalUnitPriceDiscount / ListUnitPrice * 100 } -// Prices_final_v1_3 table -.create-merge table Prices_final_v1_3 ( +// Prices_final_v1_4 table +.create-merge table Prices_final_v1_4 ( BillingAccountId: string, BillingAccountName: string, BillingCurrency: string, @@ -255,13 +255,13 @@ Prices_transform_v1_3() x_TotalUnitPriceDiscountPercent: real // Hubs add-on ) -// Update policy for Prices_raw -> Prices_final_v1_3 -.alter table Prices_final_v1_3 policy update +// Update policy for Prices_raw -> Prices_final_v1_4 +.alter table Prices_final_v1_4 policy update ``` [{ "IsEnabled": true, "Source": "Prices_raw", - "Query": "Prices_transform_v1_3()", + "Query": "Prices_transform_v1_4()", "IsTransactional": true, "PropagateIngestionProperties": true }] @@ -286,10 +286,10 @@ Prices_transform_v1_3() // Support for non-Azure data is limited to ingestion only. Data is not transformed across versions. //====================================================================================================================== -// Costs_transform_v1_3 function +// Costs_transform_v1_4 function .create-or-alter function -with (docstring='All costs transformed to FOCUS 1.3.', folder='Costs') -Costs_transform_v1_3() +with (docstring='All costs transformed to FOCUS 1.4.', folder='Costs') +Costs_transform_v1_4() { let checkString = (column: string, oldValue: string, newValue: string) { iff(oldValue == newValue, dynamic({}), bag_pack(column, oldValue)) @@ -419,7 +419,7 @@ Costs_transform_v1_3() | extend tmp_ReservationPriceLookupKey = tolower(strcat(x_BillingProfileId, substring(ChargePeriodStart, 0, 7), x_SkuMeterId, x_SkuOfferId)) | as costsWithMissingPrices | join kind=leftouter ( - Prices_final_v1_3 + Prices_final_v1_4 | extend tmp_ReservationPriceLookupKey = tolower(strcat(x_BillingProfileId, substring(x_EffectivePeriodStart, 0, 7), x_SkuMeterId, x_SkuOfferId)) | where x_SkuPriceType == 'Consumption' and tmp_ReservationPriceLookupKey in ((costsWithMissingPrices | summarize by tmp_ReservationPriceLookupKey)) | summarize ListUnitPrice = min(ListUnitPrice), ContractedUnitPrice = min(ContractedUnitPrice) by tmp_ReservationPriceLookupKey, x_PricingBlockSize, PricingUnit @@ -758,8 +758,7 @@ Costs_transform_v1_3() PricingCurrency = coalesce(PricingCurrency, x_PricingCurrency), PricingQuantity, PricingUnit, - ProviderName, - PublisherName, + // ProviderName and PublisherName removed in FOCUS 1.4. Use ServiceProviderName / HostProviderName. RegionId, RegionName, ResourceId, @@ -921,8 +920,8 @@ Costs_transform_v1_3() x_UsageType } -// Costs_final_v1_3 table -.create-merge table Costs_final_v1_3 ( +// Costs_final_v1_4 table +.create-merge table Costs_final_v1_4 ( AllocatedMethodDetails: dynamic, // FOCUS 1.3+ AllocatedMethodId: string, // FOCUS 1.3+ AllocatedResourceId: string, // FOCUS 1.3+ @@ -966,8 +965,7 @@ Costs_transform_v1_3() PricingCurrency: string, PricingQuantity: real, PricingUnit: string, - ProviderName: string, - PublisherName: string, + // ProviderName and PublisherName removed in FOCUS 1.4 RegionId: string, RegionName: string, ResourceId: string, @@ -1088,13 +1086,13 @@ Costs_transform_v1_3() x_UsageType: string // AWS 1.0 ) -// Update policy for Costs_raw -> Costs_final_v1_3 table -.alter table Costs_final_v1_3 policy update +// Update policy for Costs_raw -> Costs_final_v1_4 table +.alter table Costs_final_v1_4 policy update ``` [{ "IsEnabled": true, "Source": "Costs_raw", - "Query": "Costs_transform_v1_3()", + "Query": "Costs_transform_v1_4()", "IsTransactional": true, "PropagateIngestionProperties": true }] @@ -1106,10 +1104,10 @@ Costs_transform_v1_3() // - C360-2025-04 //====================================================================================================================== -// ActualCosts_transform_v1_3 function +// ActualCosts_transform_v1_4 function .create-or-alter function -with (docstring='ActualCost exports transformed to FOCUS 1.3.', folder='Costs') -ActualCosts_transform_v1_3() +with (docstring='ActualCost exports transformed to FOCUS 1.4.', folder='Costs') +ActualCosts_transform_v1_4() { // NOTE: All open issues and questions are tracked @ https://github.com/microsoft/finops-toolkit/issues/1111 ActualCosts_raw @@ -1316,7 +1314,7 @@ ActualCosts_transform_v1_3() [{ "IsEnabled": true, "Source": "ActualCosts_raw", - "Query": "ActualCosts_transform_v1_3()", + "Query": "ActualCosts_transform_v1_4()", "IsTransactional": true, "PropagateIngestionProperties": true }] @@ -1328,10 +1326,10 @@ ActualCosts_transform_v1_3() // - C360-2025-04 //====================================================================================================================== -// AmortizedCosts_transform_v1_3 function +// AmortizedCosts_transform_v1_4 function .create-or-alter function -with (docstring='ActualCost exports transformed to FOCUS 1.3.', folder='Costs') -AmortizedCosts_transform_v1_3() +with (docstring='ActualCost exports transformed to FOCUS 1.4.', folder='Costs') +AmortizedCosts_transform_v1_4() { // NOTE: All open issues and questions are tracked @ https://github.com/microsoft/finops-toolkit/issues/1111 AmortizedCosts_raw @@ -1537,7 +1535,7 @@ AmortizedCosts_transform_v1_3() [{ "IsEnabled": true, "Source": "AmortizedCosts_raw", - "Query": "AmortizedCosts_transform_v1_3()", + "Query": "AmortizedCosts_transform_v1_4()", "IsTransactional": true, "PropagateIngestionProperties": true }] @@ -1550,10 +1548,10 @@ AmortizedCosts_transform_v1_3() // - MS MCA reservation details: 2023-03-01 -- See https://learn.microsoft.com/azure/cost-management-billing/dataset-schema/reservation-details-mca //====================================================================================================================== -// CommitmentDiscountUsage_transform_v1_3 function +// CommitmentDiscountUsage_transform_v1_4 function .create-or-alter function -with (docstring='All commitment discount usage transformed to FOCUS 1.3. This includes reservationdeatils_raw.', folder='Commitment discounts') -CommitmentDiscountUsage_transform_v1_3() +with (docstring='All commitment discount usage transformed to FOCUS 1.4. This includes reservationdeatils_raw.', folder='Commitment discounts') +CommitmentDiscountUsage_transform_v1_4() { // NOTE: All open issues and questions are tracked @ https://github.com/microsoft/finops-toolkit/issues/1111 CommitmentDiscountUsage_raw @@ -1616,8 +1614,8 @@ CommitmentDiscountUsage_transform_v1_3() x_SourceVersion = coalesce(x_SourceVersion, iff(ProviderName == 'Microsoft', '2024-03-01', '')) } -// CommitmentDiscountUsage_final_v1_3 table -.create-merge table CommitmentDiscountUsage_final_v1_3 ( +// CommitmentDiscountUsage_final_v1_4 table +.create-merge table CommitmentDiscountUsage_final_v1_4 ( ChargePeriodEnd: datetime, // Hubs add-on ChargePeriodStart: datetime, // MS 2023-03-01 CommitmentDiscountCategory: string, // Hubs add-on @@ -1650,13 +1648,110 @@ CommitmentDiscountUsage_transform_v1_3() x_SourceVersion: string // Hubs add-on ) -// Update policy for CommitmentDiscountUsage_raw -> CommitmentDiscountUsage_final_v1_3 table -.alter table CommitmentDiscountUsage_final_v1_3 policy update +// Update policy for CommitmentDiscountUsage_raw -> CommitmentDiscountUsage_final_v1_4 table +.alter table CommitmentDiscountUsage_final_v1_4 policy update ``` [{ "IsEnabled": true, "Source": "CommitmentDiscountUsage_raw", - "Query": "CommitmentDiscountUsage_transform_v1_3()", + "Query": "CommitmentDiscountUsage_transform_v1_4()", + "IsTransactional": true, + "PropagateIngestionProperties": true +}] +``` + + +//===| Contract commitment |============================================================================================ +// Supported versions: +// - FOCUS 1.3+ (no Cost Management export shipped yet; defines the schema in advance for forward compat) +//====================================================================================================================== + +// ContractCommitment_transform_v1_4 function +.create-or-alter function +with (docstring='All contract commitments aligned to FOCUS 1.4.', folder='Contract commitments') +ContractCommitment_transform_v1_4() +{ + ContractCommitment_raw + | project + BenefitCategory = column_ifexists('BenefitCategory', ''), // FOCUS 1.4+ + BillingCurrency, + ContractCommitmentApplicability = parse_json(column_ifexists('ContractCommitmentApplicability', '')),// FOCUS 1.4+ + ContractCommitmentCategory, + ContractCommitmentCost, + ContractCommitmentId, + ContractCommitmentPeriodEnd, + ContractCommitmentPeriodStart, + ContractCommitmentQuantity, + ContractCommitmentType, + ContractCommitmentUnit, + ContractId, + ContractPeriodEnd, + ContractPeriodStart, + Created = column_ifexists('Created', datetime(null)), // FOCUS 1.4+ + DiscountPercentage = column_ifexists('DiscountPercentage', real(null)), // FOCUS 1.4+ + DurationType = column_ifexists('DurationType', ''), // FOCUS 1.4+ + FulfillmentInterval = column_ifexists('FulfillmentInterval', ''), // FOCUS 1.4+ + InvoiceIssuerName, + LastUpdated = column_ifexists('LastUpdated', datetime(null)), // FOCUS 1.4+ + LifecycleStatus = column_ifexists('LifecycleStatus', ''), // FOCUS 1.4+ + Model = column_ifexists('Model', ''), // FOCUS 1.4+ + OfferCategory = column_ifexists('OfferCategory', ''), // FOCUS 1.4+ + PaymentInterval = column_ifexists('PaymentInterval', ''), // FOCUS 1.4+ + PaymentModel = column_ifexists('PaymentModel', ''), // FOCUS 1.4+ + PaymentUpfrontPercentage = column_ifexists('PaymentUpfrontPercentage', real(null)), // FOCUS 1.4+ + PricingCurrency, + PricingCurrencyContractCommitmentCost = column_ifexists('PricingCurrencyContractCommitmentCost', real(null)), // FOCUS 1.4+ + x_IngestionTime = ingestion_time(), + x_SourceName = coalesce(x_SourceName, ''), + x_SourceProvider = coalesce(x_SourceProvider, ''), + x_SourceType = coalesce(x_SourceType, ''), + x_SourceVersion = coalesce(x_SourceVersion, '') +} + +// ContractCommitment_final_v1_4 table +.create-merge table ContractCommitment_final_v1_4 ( + BenefitCategory: string, // FOCUS 1.4+ + BillingCurrency: string, // FOCUS 1.3+ + ContractCommitmentApplicability: dynamic, // FOCUS 1.4+ (JSON) + ContractCommitmentCategory: string, // FOCUS 1.3+ + ContractCommitmentCost: real, // FOCUS 1.3+ + ContractCommitmentId: string, // FOCUS 1.3+ + ContractCommitmentPeriodEnd: datetime, // FOCUS 1.3+ + ContractCommitmentPeriodStart: datetime, // FOCUS 1.3+ + ContractCommitmentQuantity: real, // FOCUS 1.3+ + ContractCommitmentType: string, // FOCUS 1.3+ + ContractCommitmentUnit: string, // FOCUS 1.3+ + ContractId: string, // FOCUS 1.3+ + ContractPeriodEnd: datetime, // FOCUS 1.3+ + ContractPeriodStart: datetime, // FOCUS 1.3+ + Created: datetime, // FOCUS 1.4+ + DiscountPercentage: real, // FOCUS 1.4+ + DurationType: string, // FOCUS 1.4+ + FulfillmentInterval: string, // FOCUS 1.4+ + InvoiceIssuerName: string, // FOCUS 1.3+ + LastUpdated: datetime, // FOCUS 1.4+ + LifecycleStatus: string, // FOCUS 1.4+ + Model: string, // FOCUS 1.4+ + OfferCategory: string, // FOCUS 1.4+ + PaymentInterval: string, // FOCUS 1.4+ + PaymentModel: string, // FOCUS 1.4+ + PaymentUpfrontPercentage: real, // FOCUS 1.4+ + PricingCurrency: string, // FOCUS 1.3+ + PricingCurrencyContractCommitmentCost: real, // FOCUS 1.4+ + x_IngestionTime: datetime, // Hubs add-on + x_SourceName: string, // Hubs add-on + x_SourceProvider: string, // Hubs add-on + x_SourceType: string, // Hubs add-on + x_SourceVersion: string // Hubs add-on +) + +// Update policy for ContractCommitment_raw -> ContractCommitment_final_v1_4 table +.alter table ContractCommitment_final_v1_4 policy update +``` +[{ + "IsEnabled": true, + "Source": "ContractCommitment_raw", + "Query": "ContractCommitment_transform_v1_4()", "IsTransactional": true, "PropagateIngestionProperties": true }] @@ -1669,10 +1764,10 @@ CommitmentDiscountUsage_transform_v1_3() // - MS CM MCA reservation recommendations: 2023-05-01 -- See https://learn.microsoft.com/azure/cost-management-billing/dataset-schema/reservation-recommendations-mca //====================================================================================================================== -// Recommendations_transform_v1_3 function +// Recommendations_transform_v1_4 function .create-or-alter function -with (docstring='All recommendations transformed to FOCUS 1.3.', folder='Recommendations') -Recommendations_transform_v1_3() +with (docstring='All recommendations transformed to FOCUS 1.4.', folder='Recommendations') +Recommendations_transform_v1_4() { // NOTE: All open issues and questions are tracked @ https://github.com/microsoft/finops-toolkit/issues/1111 let isoMonths = (duration: string) { @@ -1777,8 +1872,8 @@ Recommendations_transform_v1_3() x_SourceVersion } -// Recommendations_final_v1_3 table -.create-merge table Recommendations_final_v1_3 ( +// Recommendations_final_v1_4 table +.create-merge table Recommendations_final_v1_4 ( ProviderName: string, ResourceId: string, ResourceName: string, @@ -1801,13 +1896,13 @@ Recommendations_transform_v1_3() x_SourceVersion: string ) -// Update policy for Recommendations_raw -> Recommendations_final_v1_3 table -.alter table Recommendations_final_v1_3 policy update +// Update policy for Recommendations_raw -> Recommendations_final_v1_4 table +.alter table Recommendations_final_v1_4 policy update ``` [{ "IsEnabled": true, "Source": "Recommendations_raw", - "Query": "Recommendations_transform_v1_3()", + "Query": "Recommendations_transform_v1_4()", "IsTransactional": true, "PropagateIngestionProperties": true }] @@ -1820,10 +1915,10 @@ Recommendations_transform_v1_3() // - MS CM MCA reservation transactions: 2023-05-01 -- See https://learn.microsoft.com/azure/cost-management-billing/dataset-schema/reservation-transactions-mca //====================================================================================================================== -// Transactions_transform_v1_3 function +// Transactions_transform_v1_4 function .create-or-alter function -with (docstring='All transactions transformed to FOCUS 1.3.', folder='Transactions') -Transactions_transform_v1_3() +with (docstring='All transactions transformed to FOCUS 1.4.', folder='Transactions') +Transactions_transform_v1_4() { // NOTE: All open issues and questions are tracked @ https://github.com/microsoft/finops-toolkit/issues/1111 let isoMonths = (duration: string) { @@ -1910,8 +2005,8 @@ Transactions_transform_v1_3() x_TransactionType = EventType } -// Transactions_final_v1_3 table -.create-merge table Transactions_final_v1_3 ( +// Transactions_final_v1_4 table +.create-merge table Transactions_final_v1_4 ( BilledCost: real, // MS CM EA+MCA 2023-05-01 BillingAccountId: string, // MS CM EA+MCA 2023-05-01 BillingAccountName: string, // MS CM EA+MCA 2023-05-01 @@ -1953,13 +2048,13 @@ Transactions_transform_v1_3() x_TransactionType: string // MS CM EA+MCA 2023-05-01 ) -// Update policy for Transactions_raw -> Transactions_final_v1_3 table -.alter table Transactions_final_v1_3 policy update +// Update policy for Transactions_raw -> Transactions_final_v1_4 table +.alter table Transactions_final_v1_4 policy update ``` [{ "IsEnabled": true, "Source": "Transactions_raw", - "Query": "Transactions_transform_v1_3()", + "Query": "Transactions_transform_v1_4()", "IsTransactional": true, "PropagateIngestionProperties": true }] From f8495a52c3ecbd32701ff39f9ee3d7ced47f61e1 Mon Sep 17 00:00:00 2001 From: Michael Flanakin Date: Tue, 26 May 2026 00:22:33 -0700 Subject: [PATCH 3/6] Address PR #2126 feedback MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Pluralize ContractCommitments_raw / _transform_v1_4 / _final_v1_4 / mapping / update policy - Add BillingPeriods_raw + IngestionSetup_v1_4 transforms/finals/policies (FOCUS 1.4 supplemental dataset) - Add InvoiceDetails_raw + IngestionSetup_v1_4 transforms/finals/policies (FOCUS 1.4 supplemental dataset) - Add 12 ContractCommitment* per-row columns + CommitmentProgramEligibilityDetails + InvoiceDetailId to Costs_raw, Costs_final_v1_4, and Costs_transform_v1_4 - Fix x_Source* annotations on ContractCommitments_raw to Hubs v1_4+ - Update Costs_raw ProviderName/PublisherName annotations to FOCUS 0.5-1.3 (removed in 1.4; kept for back compat) - Update Costs_raw Region annotation: deprecated -> removed - File #2159 for the FOCUS 1.4 native-columns architecture research 🤖 Generated with [Claude Code](https://claude.ai/claude-code) Co-Authored-By: Michael Flanakin Co-Authored-By: Claude --- .../scripts/IngestionSetup_RawTables.kql | 204 ++++++++++++++--- .../Analytics/scripts/IngestionSetup_v1_4.kql | 206 ++++++++++++++++-- 2 files changed, 363 insertions(+), 47 deletions(-) diff --git a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_RawTables.kql b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_RawTables.kql index cdc281659..dfb70fda9 100644 --- a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_RawTables.kql +++ b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_RawTables.kql @@ -272,6 +272,58 @@ .alter table AmortizedCosts_raw policy streamingingestion disable +//===| BillingPeriods |================================================================================================= +// Supported versions: +// - FOCUS 1.4+ (no Cost Management export shipped yet) +//====================================================================================================================== + +// BillingPeriods_raw table -- Create the table if it doesn't exist +.create-merge table BillingPeriods_raw ( ignore: string ) + +// BillingPeriods_raw table -- Remove all columns to allow changing column types +.alter table BillingPeriods_raw ( ignore: string ) + +// BillingPeriods_raw table -- Redefine all columns +.alter table BillingPeriods_raw ( + BillingPeriodCreated: datetime, // FOCUS 1.4+ + BillingPeriodEnd: datetime, // FOCUS 1.4+ + BillingPeriodLastUpdated: datetime, // FOCUS 1.4+ + BillingPeriodStart: datetime, // FOCUS 1.4+ + BillingPeriodStatus: string, // FOCUS 1.4+ + InvoiceIssuerName: string, // FOCUS 1.4+ + x_SourceName: string, // Hubs v1_4+ + x_SourceProvider: string, // Hubs v1_4+ + x_SourceType: string, // Hubs v1_4+ + x_SourceVersion: string // Hubs v1_4+ +) + +// BillingPeriods_raw ingestion mapping +.create-or-alter table BillingPeriods_raw ingestion parquet mapping "BillingPeriods_raw_mapping" +``` +[ + { "Column": "BillingPeriodCreated", "Properties": { "Field": "BillingPeriodCreated" } }, + { "Column": "BillingPeriodEnd", "Properties": { "Field": "BillingPeriodEnd" } }, + { "Column": "BillingPeriodLastUpdated", "Properties": { "Field": "BillingPeriodLastUpdated" } }, + { "Column": "BillingPeriodStart", "Properties": { "Field": "BillingPeriodStart" } }, + { "Column": "BillingPeriodStatus", "Properties": { "Field": "BillingPeriodStatus" } }, + { "Column": "InvoiceIssuerName", "Properties": { "Field": "InvoiceIssuerName" } }, + { "Column": "x_SourceName", "Properties": { "Field": "x_SourceName" } }, + { "Column": "x_SourceProvider", "Properties": { "Field": "x_SourceProvider" } }, + { "Column": "x_SourceType", "Properties": { "Field": "x_SourceType" } }, + { "Column": "x_SourceVersion", "Properties": { "Field": "x_SourceVersion" } } +] +``` + +// BillingPeriods_raw retention policy (clear historical data) +.alter-merge table BillingPeriods_raw policy retention softdelete = 0d recoverability = disabled + +// BillingPeriods_raw retention policy (set the user-defined retention period) +.alter-merge table BillingPeriods_raw policy retention softdelete = $$rawRetentionInDays$$d recoverability = disabled + +// Disable BillingPeriods_raw streaming ingestion (required for Fabric) +.alter table BillingPeriods_raw policy streamingingestion disable + + //===| CommitmentDiscountUsage |======================================================================================== // Supported versions: // - MS EA reservation details: 2023-03-01 -- See https://learn.microsoft.com/azure/cost-management-billing/dataset-schema/reservation-details-ea @@ -335,19 +387,19 @@ .alter table CommitmentDiscountUsage_raw policy streamingingestion disable -//===| Contract commitment |============================================================================================ +//===| Contract commitments |=========================================================================================== // Supported versions: -// - FOCUS 1.3+ (no Cost Management export shipped yet) +// - FOCUS 1.4+ (no Cost Management export shipped yet) //====================================================================================================================== -// ContractCommitment_raw table -- Create the table if it doesn't exist -.create-merge table ContractCommitment_raw ( ignore: string ) +// ContractCommitments_raw table -- Create the table if it doesn't exist +.create-merge table ContractCommitments_raw ( ignore: string ) -// ContractCommitment_raw table -- Remove all columns to allow changing column types -.alter table ContractCommitment_raw ( ignore: string ) +// ContractCommitments_raw table -- Remove all columns to allow changing column types +.alter table ContractCommitments_raw ( ignore: string ) -// ContractCommitment_raw table -- Redefine all columns -.alter table ContractCommitment_raw ( +// ContractCommitments_raw table -- Redefine all columns +.alter table ContractCommitments_raw ( BenefitCategory: string, // FOCUS 1.4+ BillingCurrency: string, // FOCUS 1.3+ ContractCommitmentApplicability: string, // FOCUS 1.4+ (JSON) @@ -376,14 +428,14 @@ PaymentUpfrontPercentage: real, // FOCUS 1.4+ PricingCurrency: string, // FOCUS 1.3+ PricingCurrencyContractCommitmentCost: real, // FOCUS 1.4+ - x_SourceName: string, // Hubs v1_3+ - x_SourceProvider: string, // Hubs v1_3+ - x_SourceType: string, // Hubs v1_3+ - x_SourceVersion: string // Hubs v1_3+ + x_SourceName: string, // Hubs v1_4+ + x_SourceProvider: string, // Hubs v1_4+ + x_SourceType: string, // Hubs v1_4+ + x_SourceVersion: string // Hubs v1_4+ ) -// ContractCommitment_raw ingestion mapping -.create-or-alter table ContractCommitment_raw ingestion parquet mapping "ContractCommitment_raw_mapping" +// ContractCommitments_raw ingestion mapping +.create-or-alter table ContractCommitments_raw ingestion parquet mapping "ContractCommitments_raw_mapping" ``` [ { "Column": "BenefitCategory", "Properties": { "Field": "BenefitCategory" } }, @@ -421,14 +473,14 @@ ] ``` -// ContractCommitment_raw retention policy (clear historical data) -.alter-merge table ContractCommitment_raw policy retention softdelete = 0d recoverability = disabled +// ContractCommitments_raw retention policy (clear historical data) +.alter-merge table ContractCommitments_raw policy retention softdelete = 0d recoverability = disabled -// ContractCommitment_raw retention policy (set the user-defined retention period) -.alter-merge table ContractCommitment_raw policy retention softdelete = $$rawRetentionInDays$$d recoverability = disabled +// ContractCommitments_raw retention policy (set the user-defined retention period) +.alter-merge table ContractCommitments_raw policy retention softdelete = $$rawRetentionInDays$$d recoverability = disabled -// Disable ContractCommitment_raw streaming ingestion (required for Fabric) -.alter table ContractCommitment_raw policy streamingingestion disable +// Disable ContractCommitments_raw streaming ingestion (required for Fabric) +.alter table ContractCommitments_raw policy streamingingestion disable //===| Costs |========================================================================================================== @@ -483,14 +535,28 @@ CommitmentDiscountQuantity: real, // FOCUS 1.1+ CommitmentDiscountStatus: string, // FOCUS 1.0+ CommitmentDiscountType: string, // FOCUS 1.0-preview+ - CommitmentDiscountUnit: string, // FOCUS 1.1+ - ConsumedQuantity: real, // FOCUS 1.0+ - ConsumedUnit: string, // FOCUS 1.0+ - ContractApplied: string, // FOCUS 1.3+ (JSON) + CommitmentDiscountUnit: string, // FOCUS 1.1+ + CommitmentProgramEligibilityDetails: string, // FOCUS 1.4+ (JSON) + ConsumedQuantity: real, // FOCUS 1.0+ + ConsumedUnit: string, // FOCUS 1.0+ + ContractApplied: string, // FOCUS 1.3+ (JSON) + ContractCommitmentBenefitCategory: string, // FOCUS 1.4+ + ContractCommitmentCreated: datetime, // FOCUS 1.4+ + ContractCommitmentDiscountPercentage: real, // FOCUS 1.4+ + ContractCommitmentDurationType: string, // FOCUS 1.4+ + ContractCommitmentFulfillmentInterval: string, // FOCUS 1.4+ + ContractCommitmentLastUpdated: datetime, // FOCUS 1.4+ + ContractCommitmentLifecycleStatus: string, // FOCUS 1.4+ + ContractCommitmentModel: string, // FOCUS 1.4+ + ContractCommitmentOfferCategory: string, // FOCUS 1.4+ + ContractCommitmentPaymentInterval: string, // FOCUS 1.4+ + ContractCommitmentPaymentModel: string, // FOCUS 1.4+ + ContractCommitmentPaymentUpfrontPercentage: real, // FOCUS 1.4+ ContractedCost: real, // FOCUS 1.0+ ContractedUnitPrice: real, // FOCUS 1.0+ EffectiveCost: real, // FOCUS 1.0-preview+ HostProviderName: string, // FOCUS 1.3+ + InvoiceDetailId: string, // FOCUS 1.4+ InvoiceId: string, // FOCUS 1.2+ InvoiceIssuerName: string, // FOCUS 0.5+ ListCost: real, // FOCUS 1.0-preview+ @@ -499,9 +565,9 @@ PricingCurrency: string, // FOCUS 1.2+ PricingQuantity: real, // FOCUS 1.0-preview+ PricingUnit: string, // FOCUS 1.0-preview+ - ProviderName: string, // FOCUS 0.5+ - PublisherName: string, // FOCUS 0.5+ - Region: string, // FOCUS 0.5-1.0-preview (deprecated) + ProviderName: string, // FOCUS 0.5-1.3 (removed in 1.4; kept for back compat) + PublisherName: string, // FOCUS 0.5-1.3 (removed in 1.4; kept for back compat) + Region: string, // FOCUS 0.5-1.0-preview (removed; kept for back compat) RegionId: string, // FOCUS 1.0+ RegionName: string, // FOCUS 1.0+ ResourceId: string, // FOCUS 0.5+ @@ -770,6 +836,90 @@ .alter table Costs_raw policy streamingingestion disable +//===| InvoiceDetails |================================================================================================= +// Supported versions: +// - FOCUS 1.4+ (no Cost Management export shipped yet) +//====================================================================================================================== + +// InvoiceDetails_raw table -- Create the table if it doesn't exist +.create-merge table InvoiceDetails_raw ( ignore: string ) + +// InvoiceDetails_raw table -- Remove all columns to allow changing column types +.alter table InvoiceDetails_raw ( ignore: string ) + +// InvoiceDetails_raw table -- Redefine all columns +.alter table InvoiceDetails_raw ( + BilledCost: real, // FOCUS 1.4+ + BillingAccountId: string, // FOCUS 1.4+ + BillingCurrency: string, // FOCUS 1.4+ + BillingPeriodEnd: datetime, // FOCUS 1.4+ + BillingPeriodStart: datetime, // FOCUS 1.4+ + ChargeCategory: string, // FOCUS 1.4+ + InvoiceDetailCreated: datetime, // FOCUS 1.4+ + InvoiceDetailDescription: string, // FOCUS 1.4+ + InvoiceDetailGrain: string, // FOCUS 1.4+ + InvoiceDetailId: string, // FOCUS 1.4+ + InvoiceDetailLastUpdated: datetime, // FOCUS 1.4+ + InvoiceId: string, // FOCUS 1.4+ + InvoiceIssueDate: datetime, // FOCUS 1.4+ + InvoiceIssueStatus: string, // FOCUS 1.4+ + InvoiceIssuerName: string, // FOCUS 1.4+ + PaymentCurrency: string, // FOCUS 1.4+ + PaymentCurrencyBilledCost: real, // FOCUS 1.4+ + PaymentCurrencyInvoiceDetailId: string, // FOCUS 1.4+ + PaymentDueDate: datetime, // FOCUS 1.4+ + PaymentTerms: string, // FOCUS 1.4+ + PurchaseOrderNumber: string, // FOCUS 1.4+ + ReferenceInvoiceId: string, // FOCUS 1.4+ + x_SourceName: string, // Hubs v1_4+ + x_SourceProvider: string, // Hubs v1_4+ + x_SourceType: string, // Hubs v1_4+ + x_SourceVersion: string // Hubs v1_4+ +) + +// InvoiceDetails_raw ingestion mapping +.create-or-alter table InvoiceDetails_raw ingestion parquet mapping "InvoiceDetails_raw_mapping" +``` +[ + { "Column": "BilledCost", "Properties": { "Field": "BilledCost" } }, + { "Column": "BillingAccountId", "Properties": { "Field": "BillingAccountId" } }, + { "Column": "BillingCurrency", "Properties": { "Field": "BillingCurrency" } }, + { "Column": "BillingPeriodEnd", "Properties": { "Field": "BillingPeriodEnd" } }, + { "Column": "BillingPeriodStart", "Properties": { "Field": "BillingPeriodStart" } }, + { "Column": "ChargeCategory", "Properties": { "Field": "ChargeCategory" } }, + { "Column": "InvoiceDetailCreated", "Properties": { "Field": "InvoiceDetailCreated" } }, + { "Column": "InvoiceDetailDescription", "Properties": { "Field": "InvoiceDetailDescription" } }, + { "Column": "InvoiceDetailGrain", "Properties": { "Field": "InvoiceDetailGrain" } }, + { "Column": "InvoiceDetailId", "Properties": { "Field": "InvoiceDetailId" } }, + { "Column": "InvoiceDetailLastUpdated", "Properties": { "Field": "InvoiceDetailLastUpdated" } }, + { "Column": "InvoiceId", "Properties": { "Field": "InvoiceId" } }, + { "Column": "InvoiceIssueDate", "Properties": { "Field": "InvoiceIssueDate" } }, + { "Column": "InvoiceIssueStatus", "Properties": { "Field": "InvoiceIssueStatus" } }, + { "Column": "InvoiceIssuerName", "Properties": { "Field": "InvoiceIssuerName" } }, + { "Column": "PaymentCurrency", "Properties": { "Field": "PaymentCurrency" } }, + { "Column": "PaymentCurrencyBilledCost", "Properties": { "Field": "PaymentCurrencyBilledCost" } }, + { "Column": "PaymentCurrencyInvoiceDetailId", "Properties": { "Field": "PaymentCurrencyInvoiceDetailId" } }, + { "Column": "PaymentDueDate", "Properties": { "Field": "PaymentDueDate" } }, + { "Column": "PaymentTerms", "Properties": { "Field": "PaymentTerms" } }, + { "Column": "PurchaseOrderNumber", "Properties": { "Field": "PurchaseOrderNumber" } }, + { "Column": "ReferenceInvoiceId", "Properties": { "Field": "ReferenceInvoiceId" } }, + { "Column": "x_SourceName", "Properties": { "Field": "x_SourceName" } }, + { "Column": "x_SourceProvider", "Properties": { "Field": "x_SourceProvider" } }, + { "Column": "x_SourceType", "Properties": { "Field": "x_SourceType" } }, + { "Column": "x_SourceVersion", "Properties": { "Field": "x_SourceVersion" } } +] +``` + +// InvoiceDetails_raw retention policy (clear historical data) +.alter-merge table InvoiceDetails_raw policy retention softdelete = 0d recoverability = disabled + +// InvoiceDetails_raw retention policy (set the user-defined retention period) +.alter-merge table InvoiceDetails_raw policy retention softdelete = $$rawRetentionInDays$$d recoverability = disabled + +// Disable InvoiceDetails_raw streaming ingestion (required for Fabric) +.alter table InvoiceDetails_raw policy streamingingestion disable + + //===| Prices |========================================================================================================= // NOTE: Must be before cost details. // diff --git a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_v1_4.kql b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_v1_4.kql index 84e8d6fcd..f525f2cf1 100644 --- a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_v1_4.kql +++ b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_v1_4.kql @@ -743,13 +743,27 @@ Costs_transform_v1_4() CommitmentDiscountStatus, CommitmentDiscountType, CommitmentDiscountUnit, + CommitmentProgramEligibilityDetails = parse_json(column_ifexists('CommitmentProgramEligibilityDetails', '')), // FOCUS 1.4+ ConsumedQuantity, ConsumedUnit, ContractApplied = parse_json(ContractApplied), // FOCUS 1.3+ + ContractCommitmentBenefitCategory = column_ifexists('ContractCommitmentBenefitCategory', ''), // FOCUS 1.4+ + ContractCommitmentCreated = column_ifexists('ContractCommitmentCreated', datetime(null)), // FOCUS 1.4+ + ContractCommitmentDiscountPercentage = column_ifexists('ContractCommitmentDiscountPercentage', real(null)), // FOCUS 1.4+ + ContractCommitmentDurationType = column_ifexists('ContractCommitmentDurationType', ''), // FOCUS 1.4+ + ContractCommitmentFulfillmentInterval = column_ifexists('ContractCommitmentFulfillmentInterval', ''), // FOCUS 1.4+ + ContractCommitmentLastUpdated = column_ifexists('ContractCommitmentLastUpdated', datetime(null)), // FOCUS 1.4+ + ContractCommitmentLifecycleStatus = column_ifexists('ContractCommitmentLifecycleStatus', ''), // FOCUS 1.4+ + ContractCommitmentModel = column_ifexists('ContractCommitmentModel', ''), // FOCUS 1.4+ + ContractCommitmentOfferCategory = column_ifexists('ContractCommitmentOfferCategory', ''), // FOCUS 1.4+ + ContractCommitmentPaymentInterval = column_ifexists('ContractCommitmentPaymentInterval', ''), // FOCUS 1.4+ + ContractCommitmentPaymentModel = column_ifexists('ContractCommitmentPaymentModel', ''), // FOCUS 1.4+ + ContractCommitmentPaymentUpfrontPercentage = column_ifexists('ContractCommitmentPaymentUpfrontPercentage', real(null)), // FOCUS 1.4+ ContractedCost = coalesce(ContractedCost, x_OnDemandCost, x_Cost), ContractedUnitPrice = coalesce(ContractedUnitPrice, x_OnDemandUnitPrice), EffectiveCost, HostProviderName = iff(isempty(HostProviderName), PublisherName, HostProviderName), // FOCUS 1.3+; falls back to deprecated PublisherName for back compat + InvoiceDetailId = column_ifexists('InvoiceDetailId', ''), // FOCUS 1.4+ InvoiceId = coalesce(InvoiceId, x_InvoiceId), InvoiceIssuerName, ListCost, @@ -949,15 +963,29 @@ Costs_transform_v1_4() CommitmentDiscountQuantity: real, CommitmentDiscountStatus: string, CommitmentDiscountType: string, - CommitmentDiscountUnit: string, - ConsumedQuantity: real, - ConsumedUnit: string, - ContractApplied: dynamic, // FOCUS 1.3+ - ContractedCost: real, - ContractedUnitPrice: real, - EffectiveCost: real, - HostProviderName: string, // FOCUS 1.3+ - InvoiceId: string, + CommitmentDiscountUnit: string, + CommitmentProgramEligibilityDetails: dynamic, // FOCUS 1.4+ (JSON) + ConsumedQuantity: real, + ConsumedUnit: string, + ContractApplied: dynamic, // FOCUS 1.3+ + ContractCommitmentBenefitCategory: string, // FOCUS 1.4+ + ContractCommitmentCreated: datetime, // FOCUS 1.4+ + ContractCommitmentDiscountPercentage: real, // FOCUS 1.4+ + ContractCommitmentDurationType: string, // FOCUS 1.4+ + ContractCommitmentFulfillmentInterval: string, // FOCUS 1.4+ + ContractCommitmentLastUpdated: datetime, // FOCUS 1.4+ + ContractCommitmentLifecycleStatus: string, // FOCUS 1.4+ + ContractCommitmentModel: string, // FOCUS 1.4+ + ContractCommitmentOfferCategory: string, // FOCUS 1.4+ + ContractCommitmentPaymentInterval: string, // FOCUS 1.4+ + ContractCommitmentPaymentModel: string, // FOCUS 1.4+ + ContractCommitmentPaymentUpfrontPercentage: real, // FOCUS 1.4+ + ContractedCost: real, + ContractedUnitPrice: real, + EffectiveCost: real, + HostProviderName: string, // FOCUS 1.3+ + InvoiceDetailId: string, // FOCUS 1.4+ + InvoiceId: string, InvoiceIssuerName: string, ListCost: real, ListUnitPrice: real, @@ -1661,17 +1689,17 @@ CommitmentDiscountUsage_transform_v1_4() ``` -//===| Contract commitment |============================================================================================ +//===| Contract commitments |=========================================================================================== // Supported versions: -// - FOCUS 1.3+ (no Cost Management export shipped yet; defines the schema in advance for forward compat) +// - FOCUS 1.4+ (no Cost Management export shipped yet; defines the schema in advance for forward compat) //====================================================================================================================== -// ContractCommitment_transform_v1_4 function +// ContractCommitments_transform_v1_4 function .create-or-alter function with (docstring='All contract commitments aligned to FOCUS 1.4.', folder='Contract commitments') -ContractCommitment_transform_v1_4() +ContractCommitments_transform_v1_4() { - ContractCommitment_raw + ContractCommitments_raw | project BenefitCategory = column_ifexists('BenefitCategory', ''), // FOCUS 1.4+ BillingCurrency, @@ -1708,8 +1736,8 @@ ContractCommitment_transform_v1_4() x_SourceVersion = coalesce(x_SourceVersion, '') } -// ContractCommitment_final_v1_4 table -.create-merge table ContractCommitment_final_v1_4 ( +// ContractCommitments_final_v1_4 table +.create-merge table ContractCommitments_final_v1_4 ( BenefitCategory: string, // FOCUS 1.4+ BillingCurrency: string, // FOCUS 1.3+ ContractCommitmentApplicability: dynamic, // FOCUS 1.4+ (JSON) @@ -1745,13 +1773,151 @@ ContractCommitment_transform_v1_4() x_SourceVersion: string // Hubs add-on ) -// Update policy for ContractCommitment_raw -> ContractCommitment_final_v1_4 table -.alter table ContractCommitment_final_v1_4 policy update +// Update policy for ContractCommitments_raw -> ContractCommitments_final_v1_4 table +.alter table ContractCommitments_final_v1_4 policy update ``` [{ "IsEnabled": true, - "Source": "ContractCommitment_raw", - "Query": "ContractCommitment_transform_v1_4()", + "Source": "ContractCommitments_raw", + "Query": "ContractCommitments_transform_v1_4()", + "IsTransactional": true, + "PropagateIngestionProperties": true +}] +``` + + +//===| BillingPeriods |================================================================================================= +// Supported versions: +// - FOCUS 1.4+ (no Cost Management export shipped yet; defines the schema in advance for forward compat) +//====================================================================================================================== + +// BillingPeriods_transform_v1_4 function +.create-or-alter function +with (docstring='All billing periods aligned to FOCUS 1.4.', folder='BillingPeriods') +BillingPeriods_transform_v1_4() +{ + BillingPeriods_raw + | project + BillingPeriodCreated = column_ifexists('BillingPeriodCreated', datetime(null)), // FOCUS 1.4+ + BillingPeriodEnd, + BillingPeriodLastUpdated = column_ifexists('BillingPeriodLastUpdated', datetime(null)), // FOCUS 1.4+ + BillingPeriodStart, + BillingPeriodStatus = column_ifexists('BillingPeriodStatus', ''), // FOCUS 1.4+ + InvoiceIssuerName, + x_IngestionTime = ingestion_time(), + x_SourceName = coalesce(x_SourceName, ''), + x_SourceProvider = coalesce(x_SourceProvider, ''), + x_SourceType = coalesce(x_SourceType, ''), + x_SourceVersion = coalesce(x_SourceVersion, '') +} + +// BillingPeriods_final_v1_4 table +.create-merge table BillingPeriods_final_v1_4 ( + BillingPeriodCreated: datetime, // FOCUS 1.4+ + BillingPeriodEnd: datetime, // FOCUS 1.4+ + BillingPeriodLastUpdated: datetime, // FOCUS 1.4+ + BillingPeriodStart: datetime, // FOCUS 1.4+ + BillingPeriodStatus: string, // FOCUS 1.4+ + InvoiceIssuerName: string, // FOCUS 1.4+ + x_IngestionTime: datetime, // Hubs add-on + x_SourceName: string, // Hubs add-on + x_SourceProvider: string, // Hubs add-on + x_SourceType: string, // Hubs add-on + x_SourceVersion: string // Hubs add-on +) + +// Update policy for BillingPeriods_raw -> BillingPeriods_final_v1_4 table +.alter table BillingPeriods_final_v1_4 policy update +``` +[{ + "IsEnabled": true, + "Source": "BillingPeriods_raw", + "Query": "BillingPeriods_transform_v1_4()", + "IsTransactional": true, + "PropagateIngestionProperties": true +}] +``` + + +//===| InvoiceDetails |================================================================================================= +// Supported versions: +// - FOCUS 1.4+ (no Cost Management export shipped yet; defines the schema in advance for forward compat) +//====================================================================================================================== + +// InvoiceDetails_transform_v1_4 function +.create-or-alter function +with (docstring='All invoice details aligned to FOCUS 1.4.', folder='InvoiceDetails') +InvoiceDetails_transform_v1_4() +{ + InvoiceDetails_raw + | project + BilledCost, + BillingAccountId, + BillingCurrency, + BillingPeriodEnd, + BillingPeriodStart, + ChargeCategory, + InvoiceDetailCreated = column_ifexists('InvoiceDetailCreated', datetime(null)), // FOCUS 1.4+ + InvoiceDetailDescription = column_ifexists('InvoiceDetailDescription', ''), // FOCUS 1.4+ + InvoiceDetailGrain = column_ifexists('InvoiceDetailGrain', ''), // FOCUS 1.4+ + InvoiceDetailId, + InvoiceDetailLastUpdated = column_ifexists('InvoiceDetailLastUpdated', datetime(null)), // FOCUS 1.4+ + InvoiceId, + InvoiceIssueDate = column_ifexists('InvoiceIssueDate', datetime(null)), // FOCUS 1.4+ + InvoiceIssueStatus = column_ifexists('InvoiceIssueStatus', ''), // FOCUS 1.4+ + InvoiceIssuerName, + PaymentCurrency = column_ifexists('PaymentCurrency', ''), // FOCUS 1.4+ + PaymentCurrencyBilledCost = column_ifexists('PaymentCurrencyBilledCost', real(null)), // FOCUS 1.4+ + PaymentCurrencyInvoiceDetailId = column_ifexists('PaymentCurrencyInvoiceDetailId', ''), // FOCUS 1.4+ + PaymentDueDate = column_ifexists('PaymentDueDate', datetime(null)), // FOCUS 1.4+ + PaymentTerms = column_ifexists('PaymentTerms', ''), // FOCUS 1.4+ + PurchaseOrderNumber = column_ifexists('PurchaseOrderNumber', ''), // FOCUS 1.4+ + ReferenceInvoiceId = column_ifexists('ReferenceInvoiceId', ''), // FOCUS 1.4+ + x_IngestionTime = ingestion_time(), + x_SourceName = coalesce(x_SourceName, ''), + x_SourceProvider = coalesce(x_SourceProvider, ''), + x_SourceType = coalesce(x_SourceType, ''), + x_SourceVersion = coalesce(x_SourceVersion, '') +} + +// InvoiceDetails_final_v1_4 table +.create-merge table InvoiceDetails_final_v1_4 ( + BilledCost: real, // FOCUS 1.4+ + BillingAccountId: string, // FOCUS 1.4+ + BillingCurrency: string, // FOCUS 1.4+ + BillingPeriodEnd: datetime, // FOCUS 1.4+ + BillingPeriodStart: datetime, // FOCUS 1.4+ + ChargeCategory: string, // FOCUS 1.4+ + InvoiceDetailCreated: datetime, // FOCUS 1.4+ + InvoiceDetailDescription: string, // FOCUS 1.4+ + InvoiceDetailGrain: string, // FOCUS 1.4+ + InvoiceDetailId: string, // FOCUS 1.4+ + InvoiceDetailLastUpdated: datetime, // FOCUS 1.4+ + InvoiceId: string, // FOCUS 1.4+ + InvoiceIssueDate: datetime, // FOCUS 1.4+ + InvoiceIssueStatus: string, // FOCUS 1.4+ + InvoiceIssuerName: string, // FOCUS 1.4+ + PaymentCurrency: string, // FOCUS 1.4+ + PaymentCurrencyBilledCost: real, // FOCUS 1.4+ + PaymentCurrencyInvoiceDetailId: string, // FOCUS 1.4+ + PaymentDueDate: datetime, // FOCUS 1.4+ + PaymentTerms: string, // FOCUS 1.4+ + PurchaseOrderNumber: string, // FOCUS 1.4+ + ReferenceInvoiceId: string, // FOCUS 1.4+ + x_IngestionTime: datetime, // Hubs add-on + x_SourceName: string, // Hubs add-on + x_SourceProvider: string, // Hubs add-on + x_SourceType: string, // Hubs add-on + x_SourceVersion: string // Hubs add-on +) + +// Update policy for InvoiceDetails_raw -> InvoiceDetails_final_v1_4 table +.alter table InvoiceDetails_final_v1_4 policy update +``` +[{ + "IsEnabled": true, + "Source": "InvoiceDetails_raw", + "Query": "InvoiceDetails_transform_v1_4()", "IsTransactional": true, "PropagateIngestionProperties": true }] From 30f926d326dad8e03ab90aa63da0a6c6dc2f96c9 Mon Sep 17 00:00:00 2001 From: Michael Flanakin Date: Wed, 15 Jul 2026 12:58:18 -0700 Subject: [PATCH 4/6] fix: address round-2 review feedback for FOCUS 1.4 ingestion - Fix swapped HostProviderName/ServiceProviderName fallbacks and guarantee non-null ServiceProviderName (C1) - Project all 22 new Costs_raw columns from the ActualCosts/AmortizedCosts C360 transforms in v1_0, v1_2, and v1_4 (C2) - Disable v1_2 update policies and add deprecation docstrings pointing to the _v1_4() transforms (C3) - Detect native FOCUS 1.3/1.4 exports in x_SourceVersion and treat 1.4 as the current version for x_SourceChanges (H1) - Always preserve removed ProviderName/PublisherName values in x_SourceValues (M1) - Apply review suggestions: raw table annotation wording, FOCUS 1.2+ comment fixes, remove stale placement comments - Split IngestionSetup_v1_4.kql into IngestionSetup_v1_4.kql (Prices + Costs) and IngestionSetup_v1_4_Datasets.kql (remaining datasets, alphabetized) to stay under the Bicep loadTextContent() 131,072-character limit; register the new file in app.bicep and .build.config Co-Authored-By: Claude Fable 5 --- src/templates/finops-hub/.build.config | 3 +- .../Microsoft.FinOpsHubs/Analytics/app.bicep | 1 + .../scripts/IngestionSetup_RawTables.kql | 6 +- .../Analytics/scripts/IngestionSetup_v1_0.kql | 44 ++ .../Analytics/scripts/IngestionSetup_v1_2.kql | 72 +- .../Analytics/scripts/IngestionSetup_v1_4.kql | 737 ++---------------- .../scripts/IngestionSetup_v1_4_Datasets.kql | 669 ++++++++++++++++ 7 files changed, 845 insertions(+), 687 deletions(-) create mode 100644 src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_v1_4_Datasets.kql diff --git a/src/templates/finops-hub/.build.config b/src/templates/finops-hub/.build.config index 59ccbdc8c..3057658b4 100644 --- a/src/templates/finops-hub/.build.config +++ b/src/templates/finops-hub/.build.config @@ -34,7 +34,8 @@ "modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_RawTables.kql", "modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_v1_0.kql", "modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_v1_2.kql", - "modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_v1_4.kql" + "modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_v1_4.kql", + "modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_v1_4_Datasets.kql" ] }, { diff --git a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/app.bicep b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/app.bicep index e20e982bd..577bd57c9 100644 --- a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/app.bicep +++ b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/app.bicep @@ -379,6 +379,7 @@ module ingestion_VersionedScripts '../../fx/hub-database.bicep' = if (useAzure) v1_0: loadTextContent('scripts/IngestionSetup_v1_0.kql') v1_2: loadTextContent('scripts/IngestionSetup_v1_2.kql') v1_4: loadTextContent('scripts/IngestionSetup_v1_4.kql') + v1_4_datasets: loadTextContent('scripts/IngestionSetup_v1_4_Datasets.kql') } continueOnErrors: continueOnErrors forceUpdateTag: forceUpdateTag diff --git a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_RawTables.kql b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_RawTables.kql index dfb70fda9..f015577f6 100644 --- a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_RawTables.kql +++ b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_RawTables.kql @@ -565,9 +565,9 @@ PricingCurrency: string, // FOCUS 1.2+ PricingQuantity: real, // FOCUS 1.0-preview+ PricingUnit: string, // FOCUS 1.0-preview+ - ProviderName: string, // FOCUS 0.5-1.3 (removed in 1.4; kept for back compat) - PublisherName: string, // FOCUS 0.5-1.3 (removed in 1.4; kept for back compat) - Region: string, // FOCUS 0.5-1.0-preview (removed; kept for back compat) + ProviderName: string, // FOCUS 0.5-1.3 (removed; keep for back-compat) + PublisherName: string, // FOCUS 0.5-1.3 (removed; keep for back-compat) + Region: string, // FOCUS 0.5-1.0-preview (removed; keep for back-compat) RegionId: string, // FOCUS 1.0+ RegionName: string, // FOCUS 1.0+ ResourceId: string, // FOCUS 0.5+ diff --git a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_v1_0.kql b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_v1_0.kql index 1484f82ab..60c52a2db 100644 --- a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_v1_0.kql +++ b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_v1_0.kql @@ -847,6 +847,11 @@ ActualCosts_transform_v1_0() | join kind=leftouter (Services | where isnotempty(x_ConsumedService) | summarize take_any(ServiceName), take_any(ServiceCategory) by ConsumedService = x_ConsumedService) on ConsumedService | extend CommitmentDiscountCategory = iff(isnotempty(ReservationId), 'Usage', '') // TODO: CommitmentDiscountCategory needs to handle savings plans | project + AllocatedMethodDetails = '', + AllocatedMethodId = '', + AllocatedResourceId = '', + AllocatedResourceName = '', + AllocatedTags = '', AvailabilityZone = AvailabilityZone, BilledCost = iff(x_AmortizationClass == 'Amortized Charge', real(0), Cost), BillingAccountId = strcat('/providers/microsoft.billing/billingaccounts/', BillingAccountId, iff(BillingAccountId == BillingProfileId or isempty(BillingProfileId), '', strcat('/billingprofiles/', BillingProfileId))), @@ -886,11 +891,27 @@ ActualCosts_transform_v1_0() ), CommitmentDiscountType = iff(isnotempty(ReservationId), 'Reservation', ''), CommitmentDiscountUnit = '', + CommitmentProgramEligibilityDetails = '', ConsumedQuantity = PricingQuantity * x_PricingBlockSize, ConsumedUnit = PricingUnit, + ContractApplied = '', + ContractCommitmentBenefitCategory = '', + ContractCommitmentCreated = datetime(null), + ContractCommitmentDiscountPercentage = real(null), + ContractCommitmentDurationType = '', + ContractCommitmentFulfillmentInterval = '', + ContractCommitmentLastUpdated = datetime(null), + ContractCommitmentLifecycleStatus = '', + ContractCommitmentModel = '', + ContractCommitmentOfferCategory = '', + ContractCommitmentPaymentInterval = '', + ContractCommitmentPaymentModel = '', + ContractCommitmentPaymentUpfrontPercentage = real(null), ContractedCost = UnitPrice * PricingQuantity, ContractedUnitPrice = UnitPrice, EffectiveCost = iff(x_AmortizationClass == 'Principal', real(0), Cost), + HostProviderName = '', + InvoiceDetailId = '', InvoiceId = '', InvoiceIssuerName = 'Microsoft', ListCost = real(null), @@ -909,6 +930,7 @@ ActualCosts_transform_v1_0() ResourceType, ServiceCategory, ServiceName, + ServiceProviderName = '', ServiceSubcategory = '', SkuId = '', SkuMeter = '', @@ -1069,6 +1091,11 @@ AmortizedCosts_transform_v1_0() | join kind=leftouter (Services | where isnotempty(x_ConsumedService) | summarize take_any(ServiceName), take_any(ServiceCategory) by ConsumedService = x_ConsumedService) on ConsumedService | extend CommitmentDiscountCategory = iff(isnotempty(ReservationId), 'Usage', '') // TODO: CommitmentDiscountCategory needs to handle savings plans | project + AllocatedMethodDetails = '', + AllocatedMethodId = '', + AllocatedResourceId = '', + AllocatedResourceName = '', + AllocatedTags = '', AvailabilityZone = AvailabilityZone, BilledCost = iff(x_AmortizationClass == 'Amortized Charge', real(0), Cost), BillingAccountId = strcat('/providers/microsoft.billing/billingaccounts/', BillingAccountId, iff(BillingAccountId == BillingProfileId or isempty(BillingProfileId), '', strcat('/billingprofiles/', BillingProfileId))), @@ -1108,11 +1135,27 @@ AmortizedCosts_transform_v1_0() ), CommitmentDiscountType = iff(isnotempty(ReservationId), 'Reservation', ''), CommitmentDiscountUnit = '', + CommitmentProgramEligibilityDetails = '', ConsumedQuantity = PricingQuantity * x_PricingBlockSize, ConsumedUnit = PricingUnit, + ContractApplied = '', + ContractCommitmentBenefitCategory = '', + ContractCommitmentCreated = datetime(null), + ContractCommitmentDiscountPercentage = real(null), + ContractCommitmentDurationType = '', + ContractCommitmentFulfillmentInterval = '', + ContractCommitmentLastUpdated = datetime(null), + ContractCommitmentLifecycleStatus = '', + ContractCommitmentModel = '', + ContractCommitmentOfferCategory = '', + ContractCommitmentPaymentInterval = '', + ContractCommitmentPaymentModel = '', + ContractCommitmentPaymentUpfrontPercentage = real(null), ContractedCost = UnitPrice * PricingQuantity, ContractedUnitPrice = UnitPrice, EffectiveCost = iff(x_AmortizationClass == 'Principal', real(0), Cost), + HostProviderName = '', + InvoiceDetailId = '', InvoiceId = '', InvoiceIssuerName = 'Microsoft', ListCost = real(null), @@ -1131,6 +1174,7 @@ AmortizedCosts_transform_v1_0() ResourceType, ServiceCategory, ServiceName, + ServiceProviderName = '', ServiceSubcategory = '', SkuId = '', SkuMeter = '', diff --git a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_v1_2.kql b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_v1_2.kql index daeab5527..0156f83f4 100644 --- a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_v1_2.kql +++ b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_v1_2.kql @@ -16,7 +16,7 @@ // Prices_transform_v1_2 function .create-or-alter function -with (docstring='Transforms Prices_raw into FOCUS 1.2.', folder='Prices') +with (docstring='DEPRECATED: Transforms Prices_raw into FOCUS 1.2. Use Prices_transform_v1_4() instead.', folder='Prices') Prices_transform_v1_2() { // NOTE: All open issues and questions are tracked @ https://github.com/microsoft/finops-toolkit/issues/1111 @@ -259,7 +259,7 @@ Prices_transform_v1_2() .alter table Prices_final_v1_2 policy update ``` [{ - "IsEnabled": true, + "IsEnabled": false, "Source": "Prices_raw", "Query": "Prices_transform_v1_2()", "IsTransactional": true, @@ -288,7 +288,7 @@ Prices_transform_v1_2() // Costs_transform_v1_2 function .create-or-alter function -with (docstring='All costs transformed to FOCUS 1.2.', folder='Costs') +with (docstring='DEPRECATED: All costs transformed to FOCUS 1.2. Use Costs_transform_v1_4() instead.', folder='Costs') Costs_transform_v1_2() { let checkString = (column: string, oldValue: string, newValue: string) { @@ -1076,7 +1076,7 @@ Costs_transform_v1_2() .alter table Costs_final_v1_2 policy update ``` [{ - "IsEnabled": true, + "IsEnabled": false, "Source": "Costs_raw", "Query": "Costs_transform_v1_2()", "IsTransactional": true, @@ -1092,7 +1092,7 @@ Costs_transform_v1_2() // ActualCosts_transform_v1_2 function .create-or-alter function -with (docstring='ActualCost exports transformed to FOCUS 1.2.', folder='Costs') +with (docstring='DEPRECATED: ActualCost exports transformed to FOCUS 1.2. Use ActualCosts_transform_v1_4() instead.', folder='Costs') ActualCosts_transform_v1_2() { // NOTE: All open issues and questions are tracked @ https://github.com/microsoft/finops-toolkit/issues/1111 @@ -1127,6 +1127,11 @@ ActualCosts_transform_v1_2() | join kind=leftouter (Services | where isnotempty(x_ConsumedService) | summarize take_any(ServiceName), take_any(ServiceCategory), take_any(ServiceSubcategory), take_any(x_ServiceModel) by ConsumedService = x_ConsumedService) on ConsumedService | extend CommitmentDiscountCategory = iff(isnotempty(ReservationId), 'Usage', '') // TODO: CommitmentDiscountCategory needs to handle savings plans | project + AllocatedMethodDetails = '', + AllocatedMethodId = '', + AllocatedResourceId = '', + AllocatedResourceName = '', + AllocatedTags = '', AvailabilityZone = AvailabilityZone, BilledCost = iff(x_AmortizationClass == 'Amortized Charge', real(0), Cost), BillingAccountId = strcat('/providers/microsoft.billing/billingaccounts/', BillingAccountId, iff(BillingAccountId == BillingProfileId or isempty(BillingProfileId), '', strcat('/billingprofiles/', BillingProfileId))), @@ -1166,11 +1171,27 @@ ActualCosts_transform_v1_2() ), CommitmentDiscountType = iff(isnotempty(ReservationId), 'Reservation', ''), CommitmentDiscountUnit = '', + CommitmentProgramEligibilityDetails = '', ConsumedQuantity = PricingQuantity * x_PricingBlockSize, ConsumedUnit = PricingUnit, + ContractApplied = '', + ContractCommitmentBenefitCategory = '', + ContractCommitmentCreated = datetime(null), + ContractCommitmentDiscountPercentage = real(null), + ContractCommitmentDurationType = '', + ContractCommitmentFulfillmentInterval = '', + ContractCommitmentLastUpdated = datetime(null), + ContractCommitmentLifecycleStatus = '', + ContractCommitmentModel = '', + ContractCommitmentOfferCategory = '', + ContractCommitmentPaymentInterval = '', + ContractCommitmentPaymentModel = '', + ContractCommitmentPaymentUpfrontPercentage = real(null), ContractedCost = UnitPrice * PricingQuantity, ContractedUnitPrice = UnitPrice, EffectiveCost = iff(x_AmortizationClass == 'Principal', real(0), Cost), + HostProviderName = '', + InvoiceDetailId = '', InvoiceId = '', InvoiceIssuerName = 'Microsoft', ListCost = real(null), @@ -1189,6 +1210,7 @@ ActualCosts_transform_v1_2() ResourceType, ServiceCategory, ServiceName, + ServiceProviderName = '', ServiceSubcategory, SkuId = '', SkuMeter = MeterName, @@ -1298,7 +1320,7 @@ ActualCosts_transform_v1_2() .alter table Costs_raw policy update ``` [{ - "IsEnabled": true, + "IsEnabled": false, "Source": "ActualCosts_raw", "Query": "ActualCosts_transform_v1_2()", "IsTransactional": true, @@ -1314,7 +1336,7 @@ ActualCosts_transform_v1_2() // AmortizedCosts_transform_v1_2 function .create-or-alter function -with (docstring='ActualCost exports transformed to FOCUS 1.2.', folder='Costs') +with (docstring='DEPRECATED: ActualCost exports transformed to FOCUS 1.2. Use AmortizedCosts_transform_v1_4() instead.', folder='Costs') AmortizedCosts_transform_v1_2() { // NOTE: All open issues and questions are tracked @ https://github.com/microsoft/finops-toolkit/issues/1111 @@ -1348,6 +1370,11 @@ AmortizedCosts_transform_v1_2() | join kind=leftouter (Services | where isnotempty(x_ConsumedService) | summarize take_any(ServiceName), take_any(ServiceCategory), take_any(ServiceSubcategory), take_any(x_ServiceModel) by ConsumedService = x_ConsumedService) on ConsumedService | extend CommitmentDiscountCategory = iff(isnotempty(ReservationId), 'Usage', '') // TODO: CommitmentDiscountCategory needs to handle savings plans | project + AllocatedMethodDetails = '', + AllocatedMethodId = '', + AllocatedResourceId = '', + AllocatedResourceName = '', + AllocatedTags = '', AvailabilityZone = AvailabilityZone, BilledCost = iff(x_AmortizationClass == 'Amortized Charge', real(0), Cost), BillingAccountId = strcat('/providers/microsoft.billing/billingaccounts/', BillingAccountId, iff(BillingAccountId == BillingProfileId or isempty(BillingProfileId), '', strcat('/billingprofiles/', BillingProfileId))), @@ -1387,11 +1414,27 @@ AmortizedCosts_transform_v1_2() ), CommitmentDiscountType = iff(isnotempty(ReservationId), 'Reservation', ''), CommitmentDiscountUnit = '', + CommitmentProgramEligibilityDetails = '', ConsumedQuantity = PricingQuantity * x_PricingBlockSize, ConsumedUnit = PricingUnit, + ContractApplied = '', + ContractCommitmentBenefitCategory = '', + ContractCommitmentCreated = datetime(null), + ContractCommitmentDiscountPercentage = real(null), + ContractCommitmentDurationType = '', + ContractCommitmentFulfillmentInterval = '', + ContractCommitmentLastUpdated = datetime(null), + ContractCommitmentLifecycleStatus = '', + ContractCommitmentModel = '', + ContractCommitmentOfferCategory = '', + ContractCommitmentPaymentInterval = '', + ContractCommitmentPaymentModel = '', + ContractCommitmentPaymentUpfrontPercentage = real(null), ContractedCost = UnitPrice * PricingQuantity, ContractedUnitPrice = UnitPrice, EffectiveCost = iff(x_AmortizationClass == 'Principal', real(0), Cost), + HostProviderName = '', + InvoiceDetailId = '', InvoiceId = '', InvoiceIssuerName = 'Microsoft', ListCost = real(null), @@ -1410,6 +1453,7 @@ AmortizedCosts_transform_v1_2() ResourceType, ServiceCategory, ServiceName, + ServiceProviderName = '', ServiceSubcategory, SkuId = '', SkuMeter = MeterName, @@ -1519,7 +1563,7 @@ AmortizedCosts_transform_v1_2() .alter table Costs_raw policy update ``` [{ - "IsEnabled": true, + "IsEnabled": false, "Source": "AmortizedCosts_raw", "Query": "AmortizedCosts_transform_v1_2()", "IsTransactional": true, @@ -1536,7 +1580,7 @@ AmortizedCosts_transform_v1_2() // CommitmentDiscountUsage_transform_v1_2 function .create-or-alter function -with (docstring='All commitment discount usage transformed to FOCUS 1.2. This includes reservationdeatils_raw.', folder='Commitment discounts') +with (docstring='DEPRECATED: All commitment discount usage transformed to FOCUS 1.2. This includes reservationdeatils_raw. Use CommitmentDiscountUsage_transform_v1_4() instead.', folder='Commitment discounts') CommitmentDiscountUsage_transform_v1_2() { // NOTE: All open issues and questions are tracked @ https://github.com/microsoft/finops-toolkit/issues/1111 @@ -1638,7 +1682,7 @@ CommitmentDiscountUsage_transform_v1_2() .alter table CommitmentDiscountUsage_final_v1_2 policy update ``` [{ - "IsEnabled": true, + "IsEnabled": false, "Source": "CommitmentDiscountUsage_raw", "Query": "CommitmentDiscountUsage_transform_v1_2()", "IsTransactional": true, @@ -1655,7 +1699,7 @@ CommitmentDiscountUsage_transform_v1_2() // Recommendations_transform_v1_2 function .create-or-alter function -with (docstring='All recommendations transformed to FOCUS 1.2.', folder='Recommendations') +with (docstring='DEPRECATED: All recommendations transformed to FOCUS 1.2. Use Recommendations_transform_v1_4() instead.', folder='Recommendations') Recommendations_transform_v1_2() { // NOTE: All open issues and questions are tracked @ https://github.com/microsoft/finops-toolkit/issues/1111 @@ -1789,7 +1833,7 @@ Recommendations_transform_v1_2() .alter table Recommendations_final_v1_2 policy update ``` [{ - "IsEnabled": true, + "IsEnabled": false, "Source": "Recommendations_raw", "Query": "Recommendations_transform_v1_2()", "IsTransactional": true, @@ -1806,7 +1850,7 @@ Recommendations_transform_v1_2() // Transactions_transform_v1_2 function .create-or-alter function -with (docstring='All transactions transformed to FOCUS 1.2.', folder='Transactions') +with (docstring='DEPRECATED: All transactions transformed to FOCUS 1.2. Use Transactions_transform_v1_4() instead.', folder='Transactions') Transactions_transform_v1_2() { // NOTE: All open issues and questions are tracked @ https://github.com/microsoft/finops-toolkit/issues/1111 @@ -1941,7 +1985,7 @@ Transactions_transform_v1_2() .alter table Transactions_final_v1_2 policy update ``` [{ - "IsEnabled": true, + "IsEnabled": false, "Source": "Transactions_raw", "Query": "Transactions_transform_v1_2()", "IsTransactional": true, diff --git a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_v1_4.kql b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_v1_4.kql index f525f2cf1..88b5f14e5 100644 --- a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_v1_4.kql +++ b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_v1_4.kql @@ -8,6 +8,10 @@ // For allowed commands, see https://learn.microsoft.com/azure/data-explorer/database-script +// NOTE: FOCUS 1.4 ingestion setup is split into IngestionSetup_v1_4.kql (Prices + Costs) and IngestionSetup_v1_4_Datasets.kql +// (other datasets) to stay under the Bicep loadTextContent() 131,072-character limit. Scripts within a deployment batch +// are not guaranteed to run in order, so each file must remain self-contained (no cross-file references). + //===| Prices |========================================================================================================= // Supported versions: // - MS EA 2023-05-01 -- See https://learn.microsoft.com/azure/cost-management-billing/dataset-schema/price-sheet-ea @@ -333,7 +337,7 @@ Costs_transform_v1_4() // )) // // Identify data quality issues - // TODO: Remove x_SourceChanges in v1_3 (or later) + // TODO: Remove x_SourceChanges in a future schema version | extend x_SourceChanges = trim_end(',', strcat( iff(ChargeCategory == 'Purchase' and isnotempty(CommitmentDiscountId) and ChargeFrequency == 'Usage-Based', 'InvalidChargeFrequency,', ''), iff(ChargeCategory == 'Purchase' and isnotempty(CommitmentDiscountId) and EffectiveCost > 0, 'InvalidEffectiveCost,', ''), @@ -383,6 +387,7 @@ Costs_transform_v1_4() | extend x_SourceProvider = coalesce(x_SourceProvider, ProviderName) | extend x_SourceType = coalesce(x_SourceType, iff(isnotempty(x_BillingProfileId), 'FocusCost', '')) | extend x_SourceVersion = coalesce(x_SourceVersion, case( + isnotempty(coalesce(ServiceProviderName, HostProviderName, InvoiceDetailId)), '1.4', isnotempty(coalesce(InvoiceId, SkuMeter, PricingCurrency)) and isempty(SkuPriceDetails) and isnotempty(x_SkuDetails), '1.2-preview', isnotempty(coalesce(InvoiceId, SkuMeter, PricingCurrency)), '1.2', isnotempty(coalesce(ChargeClass, CommitmentDiscountStatus, tostring(ConsumedQuantity), ConsumedUnit, tostring(ContractedCost), tostring(ContractedUnitPrice), RegionId, RegionName)), '1.0', @@ -390,7 +395,7 @@ Costs_transform_v1_4() '' )) // Append version check error code - | extend x_SourceChanges = iff(x_SourceVersion == '1.0', x_SourceChanges, + | extend x_SourceChanges = iff(x_SourceVersion == '1.4', x_SourceChanges, strcat(x_SourceChanges, iff(isempty(x_SourceChanges), '', ','), iff(x_SourceVersion == '', 'UnknownFocusVersion', 'LegacyFocusVersion')) ) // @@ -524,7 +529,7 @@ Costs_transform_v1_4() toreal(coalesce(x_SkuDetails.RINormalizationRatio, SkuPriceDetails.RINormalizationRatio, SkuPriceDetails.x_RINormalizationRatio, dynamic(1))) ) | extend old_CommitmentDiscountQuantity = CommitmentDiscountQuantity, CommitmentDiscountQuantity = case( - // FOCUS 1.3 + // FOCUS 1.2+ isnotempty(CommitmentDiscountQuantity), CommitmentDiscountQuantity, // FOCUS 1.0-preview, 1.0 isempty(CommitmentDiscountStatus), real(null), @@ -533,7 +538,7 @@ Costs_transform_v1_4() real(null) ) | extend old_CommitmentDiscountUnit = CommitmentDiscountUnit, CommitmentDiscountUnit = case( - // FOCUS 1.3 + // FOCUS 1.2+ isnotempty(CommitmentDiscountUnit), CommitmentDiscountUnit, // FOCUS 1.0 isempty(CommitmentDiscountQuantity), '', @@ -561,7 +566,7 @@ Costs_transform_v1_4() // // Pricing | extend old_x_AmortizationClass = x_AmortizationClass, x_AmortizationClass = case( - // FOCUS 1.3 + // FOCUS 1.2+ isnotempty(x_AmortizationClass), x_AmortizationClass, // FOCUS 1.0-preview+ ChargeCategory == 'Purchase' and (tolower(ResourceId) contains '/microsoft.capacity/reservationorders/' or tolower(ResourceId) contains '/microsoft.billingbenefits/savingsplanorders/'), 'Principal', @@ -649,7 +654,7 @@ Costs_transform_v1_4() ) | extend x_ConsumedCoreHours = iff(ConsumedUnit == 'Hours' and isnotempty(x_SkuCoreCount), x_SkuCoreCount * ConsumedQuantity, real(null)) | extend SkuPriceDetails = case( - // FOCUS 1.3 + // FOCUS 1.2+ isnotempty(SkuPriceDetails), SkuPriceDetails, // FOCUS 1.0-preview, 1.0 parse_json(replace_regex(replace_regex(replace_regex(replace_regex(replace_regex(replace_regex(tostring(x_SkuDetails) @@ -762,7 +767,7 @@ Costs_transform_v1_4() ContractedCost = coalesce(ContractedCost, x_OnDemandCost, x_Cost), ContractedUnitPrice = coalesce(ContractedUnitPrice, x_OnDemandUnitPrice), EffectiveCost, - HostProviderName = iff(isempty(HostProviderName), PublisherName, HostProviderName), // FOCUS 1.3+; falls back to deprecated PublisherName for back compat + HostProviderName = iff(isempty(HostProviderName), ProviderName, HostProviderName), // FOCUS 1.3+; falls back to deprecated ProviderName for back compat InvoiceDetailId = column_ifexists('InvoiceDetailId', ''), // FOCUS 1.4+ InvoiceId = coalesce(InvoiceId, x_InvoiceId), InvoiceIssuerName, @@ -772,7 +777,6 @@ Costs_transform_v1_4() PricingCurrency = coalesce(PricingCurrency, x_PricingCurrency), PricingQuantity, PricingUnit, - // ProviderName and PublisherName removed in FOCUS 1.4. Use ServiceProviderName / HostProviderName. RegionId, RegionName, ResourceId, @@ -780,7 +784,16 @@ Costs_transform_v1_4() ResourceType, ServiceCategory, ServiceName, - ServiceProviderName = iff(isempty(ServiceProviderName), ProviderName, ServiceProviderName), // FOCUS 1.3+; falls back to deprecated ProviderName for back compat + ServiceProviderName = case( + // FOCUS 1.3+ + isnotempty(ServiceProviderName), ServiceProviderName, + // Fall back to deprecated PublisherName for back compat (the Marketplace publisher; normalized to the cloud provider for non-Marketplace charges) + isnotempty(PublisherName), PublisherName, + // Fall back to deprecated ProviderName when PublisherName could not be resolved + isnotempty(ProviderName), ProviderName, + // Must not be null per FOCUS; Cost Management data is always Microsoft + 'Microsoft' + ), ServiceSubcategory, // TODO: Populate ServiceSubcategory from ServiceName when missing SkuId, SkuMeter = coalesce(SkuMeter, x_SkuMeterName), @@ -915,8 +928,8 @@ Costs_transform_v1_4() checkReal('ListUnitPrice', old_ListUnitPrice, ListUnitPrice), checkString('PricingCategory', old_PricingCategory, PricingCategory), checkReal('PricingQuantity', old_PricingQuantity, PricingQuantity), - checkString('ProviderName', old_ProviderName, ProviderName), - checkString('PublisherName', old_PublisherName, PublisherName), + checkString('ProviderName', old_ProviderName, ''), // Not included in final schema; use empty string + checkString('PublisherName', old_PublisherName, ''), // Not included in final schema; use empty string checkString('Region', old_Region, ''), // Not included in final schema; use empty string checkString('RegionId', old_RegionId, RegionId), checkString('ResourceId', old_ResourceId, ResourceId), @@ -993,7 +1006,6 @@ Costs_transform_v1_4() PricingCurrency: string, PricingQuantity: real, PricingUnit: string, - // ProviderName and PublisherName removed in FOCUS 1.4 RegionId: string, RegionName: string, ResourceId: string, @@ -1169,6 +1181,11 @@ ActualCosts_transform_v1_4() | join kind=leftouter (Services | where isnotempty(x_ConsumedService) | summarize take_any(ServiceName), take_any(ServiceCategory), take_any(ServiceSubcategory), take_any(x_ServiceModel) by ConsumedService = x_ConsumedService) on ConsumedService | extend CommitmentDiscountCategory = iff(isnotempty(ReservationId), 'Usage', '') // TODO: CommitmentDiscountCategory needs to handle savings plans | project + AllocatedMethodDetails = '', + AllocatedMethodId = '', + AllocatedResourceId = '', + AllocatedResourceName = '', + AllocatedTags = '', AvailabilityZone = AvailabilityZone, BilledCost = iff(x_AmortizationClass == 'Amortized Charge', real(0), Cost), BillingAccountId = strcat('/providers/microsoft.billing/billingaccounts/', BillingAccountId, iff(BillingAccountId == BillingProfileId or isempty(BillingProfileId), '', strcat('/billingprofiles/', BillingProfileId))), @@ -1208,11 +1225,27 @@ ActualCosts_transform_v1_4() ), CommitmentDiscountType = iff(isnotempty(ReservationId), 'Reservation', ''), CommitmentDiscountUnit = '', + CommitmentProgramEligibilityDetails = '', ConsumedQuantity = PricingQuantity * x_PricingBlockSize, ConsumedUnit = PricingUnit, + ContractApplied = '', + ContractCommitmentBenefitCategory = '', + ContractCommitmentCreated = datetime(null), + ContractCommitmentDiscountPercentage = real(null), + ContractCommitmentDurationType = '', + ContractCommitmentFulfillmentInterval = '', + ContractCommitmentLastUpdated = datetime(null), + ContractCommitmentLifecycleStatus = '', + ContractCommitmentModel = '', + ContractCommitmentOfferCategory = '', + ContractCommitmentPaymentInterval = '', + ContractCommitmentPaymentModel = '', + ContractCommitmentPaymentUpfrontPercentage = real(null), ContractedCost = UnitPrice * PricingQuantity, ContractedUnitPrice = UnitPrice, EffectiveCost = iff(x_AmortizationClass == 'Principal', real(0), Cost), + HostProviderName = '', + InvoiceDetailId = '', InvoiceId = '', InvoiceIssuerName = 'Microsoft', ListCost = real(null), @@ -1231,6 +1264,7 @@ ActualCosts_transform_v1_4() ResourceType, ServiceCategory, ServiceName, + ServiceProviderName = '', ServiceSubcategory, SkuId = '', SkuMeter = MeterName, @@ -1390,6 +1424,11 @@ AmortizedCosts_transform_v1_4() | join kind=leftouter (Services | where isnotempty(x_ConsumedService) | summarize take_any(ServiceName), take_any(ServiceCategory), take_any(ServiceSubcategory), take_any(x_ServiceModel) by ConsumedService = x_ConsumedService) on ConsumedService | extend CommitmentDiscountCategory = iff(isnotempty(ReservationId), 'Usage', '') // TODO: CommitmentDiscountCategory needs to handle savings plans | project + AllocatedMethodDetails = '', + AllocatedMethodId = '', + AllocatedResourceId = '', + AllocatedResourceName = '', + AllocatedTags = '', AvailabilityZone = AvailabilityZone, BilledCost = iff(x_AmortizationClass == 'Amortized Charge', real(0), Cost), BillingAccountId = strcat('/providers/microsoft.billing/billingaccounts/', BillingAccountId, iff(BillingAccountId == BillingProfileId or isempty(BillingProfileId), '', strcat('/billingprofiles/', BillingProfileId))), @@ -1429,11 +1468,27 @@ AmortizedCosts_transform_v1_4() ), CommitmentDiscountType = iff(isnotempty(ReservationId), 'Reservation', ''), CommitmentDiscountUnit = '', + CommitmentProgramEligibilityDetails = '', ConsumedQuantity = PricingQuantity * x_PricingBlockSize, ConsumedUnit = PricingUnit, + ContractApplied = '', + ContractCommitmentBenefitCategory = '', + ContractCommitmentCreated = datetime(null), + ContractCommitmentDiscountPercentage = real(null), + ContractCommitmentDurationType = '', + ContractCommitmentFulfillmentInterval = '', + ContractCommitmentLastUpdated = datetime(null), + ContractCommitmentLifecycleStatus = '', + ContractCommitmentModel = '', + ContractCommitmentOfferCategory = '', + ContractCommitmentPaymentInterval = '', + ContractCommitmentPaymentModel = '', + ContractCommitmentPaymentUpfrontPercentage = real(null), ContractedCost = UnitPrice * PricingQuantity, ContractedUnitPrice = UnitPrice, EffectiveCost = iff(x_AmortizationClass == 'Principal', real(0), Cost), + HostProviderName = '', + InvoiceDetailId = '', InvoiceId = '', InvoiceIssuerName = 'Microsoft', ListCost = real(null), @@ -1452,6 +1507,7 @@ AmortizedCosts_transform_v1_4() ResourceType, ServiceCategory, ServiceName, + ServiceProviderName = '', ServiceSubcategory, SkuId = '', SkuMeter = MeterName, @@ -1568,660 +1624,3 @@ AmortizedCosts_transform_v1_4() "PropagateIngestionProperties": true }] ``` - - -//===| CommitmentDiscountUsage |======================================================================================== -// Supported versions: -// - MS EA reservation details: 2023-03-01 -- See https://learn.microsoft.com/azure/cost-management-billing/dataset-schema/reservation-details-ea -// - MS MCA reservation details: 2023-03-01 -- See https://learn.microsoft.com/azure/cost-management-billing/dataset-schema/reservation-details-mca -//====================================================================================================================== - -// CommitmentDiscountUsage_transform_v1_4 function -.create-or-alter function -with (docstring='All commitment discount usage transformed to FOCUS 1.4. This includes reservationdeatils_raw.', folder='Commitment discounts') -CommitmentDiscountUsage_transform_v1_4() -{ - // NOTE: All open issues and questions are tracked @ https://github.com/microsoft/finops-toolkit/issues/1111 - CommitmentDiscountUsage_raw - // - // Set ProviderName - | extend ProviderName = 'Microsoft' - // - // Handle resource columns - | extend ResourceId = tolower(InstanceId) - | extend tmp_ResourceDetails = parse_resourceid(ResourceId) - | extend ResourceName = tostring(tmp_ResourceDetails.ResourceName) - | extend SubAccountId = tostring(tmp_ResourceDetails.SubAccountId) - | extend x_ResourceGroupName = tostring(tmp_ResourceDetails.x_ResourceGroupName) - | extend x_ResourceType = tostring(tmp_ResourceDetails.x_ResourceType) - | lookup kind=leftouter (ResourceTypes | distinct x_ResourceType, ResourceType = SingularDisplayName) on x_ResourceType - | lookup kind=leftouter (Services | distinct x_ResourceType, ServiceName, ServiceCategory, ServiceSubcategory, x_ServiceModel) on x_ResourceType - // - // Sort columns and apply final transforms - | project - ChargePeriodEnd = UsageDate + 1d, - ChargePeriodStart = UsageDate, - CommitmentDiscountCategory = 'Usage', - CommitmentDiscountId = tolower(strcat('/providers/microsoft.capacity/reservationorders/', ReservationOrderId, '/reservations/', ReservationId)), - CommitmentDiscountQuantity = UsedHours * InstanceFlexibilityRatio, - CommitmentDiscountType = 'Reservation', - CommitmentDiscountUnit = case( - InstanceFlexibilityRatio == 1, 'Hours', - InstanceFlexibilityRatio != 1, 'Normalized Hours', - '' - ), - ConsumedQuantity = UsedHours, - ProviderName, - ResourceId, - ResourceName, - ResourceType, - ServiceCategory, - ServiceName, - ServiceSubcategory, - SubAccountId, - x_CommitmentDiscountCommittedCount = TotalReservedQuantity, - x_CommitmentDiscountCommittedAmount = ReservedHours, - // TODO: Is this needed? -- x_CommitmentDiscountKind = Kind, - x_CommitmentDiscountNormalizedGroup = iff(InstanceFlexibilityGroup == 'NA', '', InstanceFlexibilityGroup), - x_CommitmentDiscountNormalizedRatio = InstanceFlexibilityRatio, - x_IngestionTime = ingestion_time(), - x_ResourceGroupName, - x_ResourceType, - // x_RowId = hash_sha256(strcat( - // // DO NOT CHANGE COLUMNS OR COLUMN ORDER - // CommitmentDiscountId, - // ResourceId, - // ChargePeriodStart - // )), - x_ServiceModel, - x_SkuOrderId = ReservationOrderId, - x_SkuSize = iff(SkuName == 'NA', '', SkuName), - x_SourceName = coalesce(x_SourceName, iff(ProviderName == 'Microsoft', 'Cost Management', ProviderName)), - x_SourceProvider = coalesce(x_SourceProvider, ProviderName), - x_SourceType = coalesce(x_SourceType, iff(ProviderName == 'Microsoft', 'ReservationDetails', '')), - x_SourceVersion = coalesce(x_SourceVersion, iff(ProviderName == 'Microsoft', '2024-03-01', '')) -} - -// CommitmentDiscountUsage_final_v1_4 table -.create-merge table CommitmentDiscountUsage_final_v1_4 ( - ChargePeriodEnd: datetime, // Hubs add-on - ChargePeriodStart: datetime, // MS 2023-03-01 - CommitmentDiscountCategory: string, // Hubs add-on - CommitmentDiscountId: string, // MS 2023-03-01 - CommitmentDiscountQuantity: real, // MS 2023-03-01 - CommitmentDiscountType: string, // Hubs add-on - CommitmentDiscountUnit: string, // Hubs add-on - ConsumedQuantity: real, // MS 2023-03-01 - ProviderName: string, // Hubs add-on - ResourceId: string, // MS 2023-03-01 - ResourceName: string, // Hubs add-on - ResourceType: string, // Hubs add-on - ServiceCategory: string, // Hubs add-on - ServiceName: string, // Hubs add-on - ServiceSubcategory: string, // Hubs add-on - SubAccountId: string, // Hubs add-on - x_CommitmentDiscountCommittedCount: real, // MS 2023-03-01 - x_CommitmentDiscountCommittedAmount: real, // MS 2023-03-01 - x_CommitmentDiscountNormalizedGroup: string, // MS 2023-03-01 - x_CommitmentDiscountNormalizedRatio: real, // MS 2023-03-01 - x_IngestionTime: datetime, // Hubs add-on - x_ResourceGroupName: string, // Hubs add-on - x_ResourceType: string, // Hubs add-on - x_ServiceModel: string, // Hubs add-on - x_SkuOrderId: string, // MS 2023-03-01 - x_SkuSize: string, // MS 2023-03-01 - x_SourceName: string, // Hubs add-on - x_SourceProvider: string, // Hubs add-on - x_SourceType: string, // Hubs add-on - x_SourceVersion: string // Hubs add-on -) - -// Update policy for CommitmentDiscountUsage_raw -> CommitmentDiscountUsage_final_v1_4 table -.alter table CommitmentDiscountUsage_final_v1_4 policy update -``` -[{ - "IsEnabled": true, - "Source": "CommitmentDiscountUsage_raw", - "Query": "CommitmentDiscountUsage_transform_v1_4()", - "IsTransactional": true, - "PropagateIngestionProperties": true -}] -``` - - -//===| Contract commitments |=========================================================================================== -// Supported versions: -// - FOCUS 1.4+ (no Cost Management export shipped yet; defines the schema in advance for forward compat) -//====================================================================================================================== - -// ContractCommitments_transform_v1_4 function -.create-or-alter function -with (docstring='All contract commitments aligned to FOCUS 1.4.', folder='Contract commitments') -ContractCommitments_transform_v1_4() -{ - ContractCommitments_raw - | project - BenefitCategory = column_ifexists('BenefitCategory', ''), // FOCUS 1.4+ - BillingCurrency, - ContractCommitmentApplicability = parse_json(column_ifexists('ContractCommitmentApplicability', '')),// FOCUS 1.4+ - ContractCommitmentCategory, - ContractCommitmentCost, - ContractCommitmentId, - ContractCommitmentPeriodEnd, - ContractCommitmentPeriodStart, - ContractCommitmentQuantity, - ContractCommitmentType, - ContractCommitmentUnit, - ContractId, - ContractPeriodEnd, - ContractPeriodStart, - Created = column_ifexists('Created', datetime(null)), // FOCUS 1.4+ - DiscountPercentage = column_ifexists('DiscountPercentage', real(null)), // FOCUS 1.4+ - DurationType = column_ifexists('DurationType', ''), // FOCUS 1.4+ - FulfillmentInterval = column_ifexists('FulfillmentInterval', ''), // FOCUS 1.4+ - InvoiceIssuerName, - LastUpdated = column_ifexists('LastUpdated', datetime(null)), // FOCUS 1.4+ - LifecycleStatus = column_ifexists('LifecycleStatus', ''), // FOCUS 1.4+ - Model = column_ifexists('Model', ''), // FOCUS 1.4+ - OfferCategory = column_ifexists('OfferCategory', ''), // FOCUS 1.4+ - PaymentInterval = column_ifexists('PaymentInterval', ''), // FOCUS 1.4+ - PaymentModel = column_ifexists('PaymentModel', ''), // FOCUS 1.4+ - PaymentUpfrontPercentage = column_ifexists('PaymentUpfrontPercentage', real(null)), // FOCUS 1.4+ - PricingCurrency, - PricingCurrencyContractCommitmentCost = column_ifexists('PricingCurrencyContractCommitmentCost', real(null)), // FOCUS 1.4+ - x_IngestionTime = ingestion_time(), - x_SourceName = coalesce(x_SourceName, ''), - x_SourceProvider = coalesce(x_SourceProvider, ''), - x_SourceType = coalesce(x_SourceType, ''), - x_SourceVersion = coalesce(x_SourceVersion, '') -} - -// ContractCommitments_final_v1_4 table -.create-merge table ContractCommitments_final_v1_4 ( - BenefitCategory: string, // FOCUS 1.4+ - BillingCurrency: string, // FOCUS 1.3+ - ContractCommitmentApplicability: dynamic, // FOCUS 1.4+ (JSON) - ContractCommitmentCategory: string, // FOCUS 1.3+ - ContractCommitmentCost: real, // FOCUS 1.3+ - ContractCommitmentId: string, // FOCUS 1.3+ - ContractCommitmentPeriodEnd: datetime, // FOCUS 1.3+ - ContractCommitmentPeriodStart: datetime, // FOCUS 1.3+ - ContractCommitmentQuantity: real, // FOCUS 1.3+ - ContractCommitmentType: string, // FOCUS 1.3+ - ContractCommitmentUnit: string, // FOCUS 1.3+ - ContractId: string, // FOCUS 1.3+ - ContractPeriodEnd: datetime, // FOCUS 1.3+ - ContractPeriodStart: datetime, // FOCUS 1.3+ - Created: datetime, // FOCUS 1.4+ - DiscountPercentage: real, // FOCUS 1.4+ - DurationType: string, // FOCUS 1.4+ - FulfillmentInterval: string, // FOCUS 1.4+ - InvoiceIssuerName: string, // FOCUS 1.3+ - LastUpdated: datetime, // FOCUS 1.4+ - LifecycleStatus: string, // FOCUS 1.4+ - Model: string, // FOCUS 1.4+ - OfferCategory: string, // FOCUS 1.4+ - PaymentInterval: string, // FOCUS 1.4+ - PaymentModel: string, // FOCUS 1.4+ - PaymentUpfrontPercentage: real, // FOCUS 1.4+ - PricingCurrency: string, // FOCUS 1.3+ - PricingCurrencyContractCommitmentCost: real, // FOCUS 1.4+ - x_IngestionTime: datetime, // Hubs add-on - x_SourceName: string, // Hubs add-on - x_SourceProvider: string, // Hubs add-on - x_SourceType: string, // Hubs add-on - x_SourceVersion: string // Hubs add-on -) - -// Update policy for ContractCommitments_raw -> ContractCommitments_final_v1_4 table -.alter table ContractCommitments_final_v1_4 policy update -``` -[{ - "IsEnabled": true, - "Source": "ContractCommitments_raw", - "Query": "ContractCommitments_transform_v1_4()", - "IsTransactional": true, - "PropagateIngestionProperties": true -}] -``` - - -//===| BillingPeriods |================================================================================================= -// Supported versions: -// - FOCUS 1.4+ (no Cost Management export shipped yet; defines the schema in advance for forward compat) -//====================================================================================================================== - -// BillingPeriods_transform_v1_4 function -.create-or-alter function -with (docstring='All billing periods aligned to FOCUS 1.4.', folder='BillingPeriods') -BillingPeriods_transform_v1_4() -{ - BillingPeriods_raw - | project - BillingPeriodCreated = column_ifexists('BillingPeriodCreated', datetime(null)), // FOCUS 1.4+ - BillingPeriodEnd, - BillingPeriodLastUpdated = column_ifexists('BillingPeriodLastUpdated', datetime(null)), // FOCUS 1.4+ - BillingPeriodStart, - BillingPeriodStatus = column_ifexists('BillingPeriodStatus', ''), // FOCUS 1.4+ - InvoiceIssuerName, - x_IngestionTime = ingestion_time(), - x_SourceName = coalesce(x_SourceName, ''), - x_SourceProvider = coalesce(x_SourceProvider, ''), - x_SourceType = coalesce(x_SourceType, ''), - x_SourceVersion = coalesce(x_SourceVersion, '') -} - -// BillingPeriods_final_v1_4 table -.create-merge table BillingPeriods_final_v1_4 ( - BillingPeriodCreated: datetime, // FOCUS 1.4+ - BillingPeriodEnd: datetime, // FOCUS 1.4+ - BillingPeriodLastUpdated: datetime, // FOCUS 1.4+ - BillingPeriodStart: datetime, // FOCUS 1.4+ - BillingPeriodStatus: string, // FOCUS 1.4+ - InvoiceIssuerName: string, // FOCUS 1.4+ - x_IngestionTime: datetime, // Hubs add-on - x_SourceName: string, // Hubs add-on - x_SourceProvider: string, // Hubs add-on - x_SourceType: string, // Hubs add-on - x_SourceVersion: string // Hubs add-on -) - -// Update policy for BillingPeriods_raw -> BillingPeriods_final_v1_4 table -.alter table BillingPeriods_final_v1_4 policy update -``` -[{ - "IsEnabled": true, - "Source": "BillingPeriods_raw", - "Query": "BillingPeriods_transform_v1_4()", - "IsTransactional": true, - "PropagateIngestionProperties": true -}] -``` - - -//===| InvoiceDetails |================================================================================================= -// Supported versions: -// - FOCUS 1.4+ (no Cost Management export shipped yet; defines the schema in advance for forward compat) -//====================================================================================================================== - -// InvoiceDetails_transform_v1_4 function -.create-or-alter function -with (docstring='All invoice details aligned to FOCUS 1.4.', folder='InvoiceDetails') -InvoiceDetails_transform_v1_4() -{ - InvoiceDetails_raw - | project - BilledCost, - BillingAccountId, - BillingCurrency, - BillingPeriodEnd, - BillingPeriodStart, - ChargeCategory, - InvoiceDetailCreated = column_ifexists('InvoiceDetailCreated', datetime(null)), // FOCUS 1.4+ - InvoiceDetailDescription = column_ifexists('InvoiceDetailDescription', ''), // FOCUS 1.4+ - InvoiceDetailGrain = column_ifexists('InvoiceDetailGrain', ''), // FOCUS 1.4+ - InvoiceDetailId, - InvoiceDetailLastUpdated = column_ifexists('InvoiceDetailLastUpdated', datetime(null)), // FOCUS 1.4+ - InvoiceId, - InvoiceIssueDate = column_ifexists('InvoiceIssueDate', datetime(null)), // FOCUS 1.4+ - InvoiceIssueStatus = column_ifexists('InvoiceIssueStatus', ''), // FOCUS 1.4+ - InvoiceIssuerName, - PaymentCurrency = column_ifexists('PaymentCurrency', ''), // FOCUS 1.4+ - PaymentCurrencyBilledCost = column_ifexists('PaymentCurrencyBilledCost', real(null)), // FOCUS 1.4+ - PaymentCurrencyInvoiceDetailId = column_ifexists('PaymentCurrencyInvoiceDetailId', ''), // FOCUS 1.4+ - PaymentDueDate = column_ifexists('PaymentDueDate', datetime(null)), // FOCUS 1.4+ - PaymentTerms = column_ifexists('PaymentTerms', ''), // FOCUS 1.4+ - PurchaseOrderNumber = column_ifexists('PurchaseOrderNumber', ''), // FOCUS 1.4+ - ReferenceInvoiceId = column_ifexists('ReferenceInvoiceId', ''), // FOCUS 1.4+ - x_IngestionTime = ingestion_time(), - x_SourceName = coalesce(x_SourceName, ''), - x_SourceProvider = coalesce(x_SourceProvider, ''), - x_SourceType = coalesce(x_SourceType, ''), - x_SourceVersion = coalesce(x_SourceVersion, '') -} - -// InvoiceDetails_final_v1_4 table -.create-merge table InvoiceDetails_final_v1_4 ( - BilledCost: real, // FOCUS 1.4+ - BillingAccountId: string, // FOCUS 1.4+ - BillingCurrency: string, // FOCUS 1.4+ - BillingPeriodEnd: datetime, // FOCUS 1.4+ - BillingPeriodStart: datetime, // FOCUS 1.4+ - ChargeCategory: string, // FOCUS 1.4+ - InvoiceDetailCreated: datetime, // FOCUS 1.4+ - InvoiceDetailDescription: string, // FOCUS 1.4+ - InvoiceDetailGrain: string, // FOCUS 1.4+ - InvoiceDetailId: string, // FOCUS 1.4+ - InvoiceDetailLastUpdated: datetime, // FOCUS 1.4+ - InvoiceId: string, // FOCUS 1.4+ - InvoiceIssueDate: datetime, // FOCUS 1.4+ - InvoiceIssueStatus: string, // FOCUS 1.4+ - InvoiceIssuerName: string, // FOCUS 1.4+ - PaymentCurrency: string, // FOCUS 1.4+ - PaymentCurrencyBilledCost: real, // FOCUS 1.4+ - PaymentCurrencyInvoiceDetailId: string, // FOCUS 1.4+ - PaymentDueDate: datetime, // FOCUS 1.4+ - PaymentTerms: string, // FOCUS 1.4+ - PurchaseOrderNumber: string, // FOCUS 1.4+ - ReferenceInvoiceId: string, // FOCUS 1.4+ - x_IngestionTime: datetime, // Hubs add-on - x_SourceName: string, // Hubs add-on - x_SourceProvider: string, // Hubs add-on - x_SourceType: string, // Hubs add-on - x_SourceVersion: string // Hubs add-on -) - -// Update policy for InvoiceDetails_raw -> InvoiceDetails_final_v1_4 table -.alter table InvoiceDetails_final_v1_4 policy update -``` -[{ - "IsEnabled": true, - "Source": "InvoiceDetails_raw", - "Query": "InvoiceDetails_transform_v1_4()", - "IsTransactional": true, - "PropagateIngestionProperties": true -}] -``` - - -//===| Recommendations |================================================================================================ -// Supported datasets/versions: -// - MS CM EA reservation recommendations: 2023-05-01 -- See https://learn.microsoft.com/azure/cost-management-billing/dataset-schema/reservation-recommendations-ea -// - MS CM MCA reservation recommendations: 2023-05-01 -- See https://learn.microsoft.com/azure/cost-management-billing/dataset-schema/reservation-recommendations-mca -//====================================================================================================================== - -// Recommendations_transform_v1_4 function -.create-or-alter function -with (docstring='All recommendations transformed to FOCUS 1.4.', folder='Recommendations') -Recommendations_transform_v1_4() -{ - // NOTE: All open issues and questions are tracked @ https://github.com/microsoft/finops-toolkit/issues/1111 - let isoMonths = (duration: string) { - let number = toint(replace_regex(duration, @'[PMY]', '')); - toint(case( - duration == '', int(null), - duration endswith "Y", number * 12, - duration endswith "M", number, - -1 - )) - }; - Recommendations_raw - | extend x_IngestionTime = ingestion_time() - // - // Set ProviderName - | extend ProviderName = 'Microsoft' - // - // Set source columns - | extend x_SourceName = coalesce(x_SourceName, iff(ProviderName == 'Microsoft', 'Cost Management', ProviderName)) - | extend x_SourceProvider = coalesce(x_SourceProvider, ProviderName) - | extend x_SourceType = coalesce(x_SourceType, iff(ProviderName == 'Microsoft', 'ReservationRecommendations', '')) - | extend x_SourceVersion = coalesce(x_SourceVersion, iff(ProviderName == 'Microsoft', '2023-05-01', '')) - // - // Convert JSON cost columns to real - | extend CostWithNoReservedInstances = case(isnotempty(CostWithNoReservedInstances), CostWithNoReservedInstances, isnotempty(CostWithNoReservedInstancesJson), toreal(extract(@'"value":([0-9\.]+)', 1, CostWithNoReservedInstancesJson)), CostWithNoReservedInstances) - | extend NetSavings = case(isnotempty(NetSavings), NetSavings, isnotempty(NetSavingsJson), toreal(extract(@'"value":([0-9\.]+)', 1, NetSavingsJson)), NetSavings) - | extend TotalCostWithReservedInstances = case(isnotempty(TotalCostWithReservedInstances), TotalCostWithReservedInstances, isnotempty(TotalCostWithReservedInstancesJson), toreal(extract(@'"value":([0-9\.]+)', 1, TotalCostWithReservedInstancesJson)), TotalCostWithReservedInstances) - // - // Parse x_RecommendationDetails from JSON string (ARG queries serialize to string for Parquet compatibility) - | extend x_RecommendationDetails = iff(gettype(x_RecommendationDetails) == 'string', parse_json(tostring(x_RecommendationDetails)), x_RecommendationDetails) - // - // Normalize x_RecommendationDetails keys to x_PascalCase (Advisor extendedProperties use camelCase) - // Guard: inject a placeholder key so mv-apply doesn't drop rows with null/empty bags - | extend x_RecommendationDetails = bag_merge(coalesce(x_RecommendationDetails, dynamic({})), bag_pack('__placeholder', '')) - | mv-apply k = bag_keys(x_RecommendationDetails) on ( - where isnotempty(tostring(k)) and tostring(k) != '__placeholder' - | extend newKey = iff(tostring(k) startswith 'x_', tostring(k), strcat('x_', toupper(substring(tostring(k), 0, 1)), substring(tostring(k), 1))) - | summarize x_RecommendationDetails = make_bag(bag_pack(newKey, x_RecommendationDetails[tostring(k)])) - ) - // - // Build recommendation details - | lookup kind=leftouter (Regions | summarize RegionName = make_set(RegionName)[0] by Location = RegionId) on Location - | extend x_RecommendationDetails = case( - // Use incoming x_RecommendationDetails first - isnotempty(x_RecommendationDetails), x_RecommendationDetails, - // Create one for reservation recommendations if needed - x_SourceType == 'ReservationRecommendations', bag_pack( - 'CommitmentDiscountNormalizedGroup', InstanceFlexibilityGroup, - 'CommitmentDiscountNormalizedRatio', InstanceFlexibilityRatio, - 'CommitmentDiscountNormalizedSize', NormalizedSize, - 'CommitmentDiscountResourceType', ResourceType, - 'CommitmentDiscountScope', Scope, - 'LookbackPeriodDuration', case( - LookBackPeriod matches regex @'^Last([0-9]+)Days$', replace_regex(LookBackPeriod, @'^Last([0-9]+)Days$', @'P\1D'), - LookBackPeriod matches regex @'^[0-9]+$', strcat('P', LookBackPeriod, 'D'), - '' - ), - 'LookbackPeriodStart', FirstUsageDate, - 'RecommendedQuantity', RecommendedQuantity, - 'RecommendedQuantityNormalized', RecommendedQuantityNormalized, - 'RegionId', Location, - 'RegionName', RegionName, - 'SkuMeterId', MeterId, - 'SkuPriceDetails', SkuProperties, - 'SkuSize', coalesce(SKU, SkuName), - 'SkuTerm', isoMonths(Term) - ), - dynamic({}) - ) - // - // Prefer specified date, then fall back to generating a date based on reservation recommendation lookback period, then validate to ensure it's not in the future - | extend x_RecommendationDate = coalesce(x_RecommendationDate, FirstUsageDate + (toint(extract(@'^P([0-9]+)D$', 1, tostring(x_RecommendationDetails.LookbackPeriodDuration))) * 1d)) - | extend x_RecommendationDate = iff(x_RecommendationDate > now(), startofday(now()), x_RecommendationDate) - // - // Derive x_ResourceType from ResourceId - | extend tmp_ResourceType = tostring(parse_resourceid(ResourceId).x_ResourceType) - | extend x_RecommendationDetails = iff(isnotempty(tmp_ResourceType), bag_merge(bag_pack('x_ResourceType', tmp_ResourceType), x_RecommendationDetails), x_RecommendationDetails) - // - // Set ResourceType display name from x_ResourceType code - | extend ResourceType = coalesce(ResourceType, tostring(resource_type(tmp_ResourceType).SingularDisplayName), tmp_ResourceType) - // - | project - ProviderName, - ResourceId = tolower(ResourceId), // Force lowercase for consistent grouping/filtering - ResourceName = tolower(iff(tmp_ResourceType =~ 'microsoft.resources/subscriptions', coalesce(SubAccountName, ResourceName), ResourceName)), // Use subscription name for subscription-level recommendations - ResourceType, - SubAccountId = coalesce(SubAccountId, iff(isnotempty(SubscriptionId), strcat('/subscriptions/', SubscriptionId), '')), - SubAccountName, - x_EffectiveCostAfter = coalesce(x_EffectiveCostAfter, TotalCostWithReservedInstances), - x_EffectiveCostBefore = coalesce(x_EffectiveCostBefore, CostWithNoReservedInstances), - x_EffectiveCostSavings = coalesce(x_EffectiveCostSavings, NetSavings, toreal(x_RecommendationDetails.x_SavingsAmount), toreal(x_RecommendationDetails.x_AnnualSavingsAmount) / 12), - x_IngestionTime, - x_RecommendationCategory, // TODO: Set for reservation recommendations - x_RecommendationDate, - x_RecommendationDescription = coalesce(x_RecommendationDescription, tostring(x_RecommendationDetails.x_RecommendationSolution), tostring(x_RecommendationDetails.x_RecommendationSubCategory)), - x_RecommendationDetails, - x_RecommendationId = tolower(x_RecommendationId), // TODO: Set for reservation recommendations; force lowercase for consistent grouping/filtering - x_ResourceGroupName = tolower(x_ResourceGroupName), // Force lowercase for consistent grouping/filtering - x_SourceName, - x_SourceProvider, - x_SourceType, - x_SourceVersion -} - -// Recommendations_final_v1_4 table -.create-merge table Recommendations_final_v1_4 ( - ProviderName: string, - ResourceId: string, - ResourceName: string, - ResourceType: string, - SubAccountId: string, - SubAccountName: string, - x_EffectiveCostAfter: real, - x_EffectiveCostBefore: real, - x_EffectiveCostSavings: real, - x_IngestionTime: datetime, - x_RecommendationCategory: string, - x_RecommendationDate: datetime, - x_RecommendationDescription: string, - x_RecommendationDetails: dynamic, - x_RecommendationId: string, - x_ResourceGroupName: string, - x_SourceName: string, - x_SourceProvider: string, - x_SourceType: string, - x_SourceVersion: string -) - -// Update policy for Recommendations_raw -> Recommendations_final_v1_4 table -.alter table Recommendations_final_v1_4 policy update -``` -[{ - "IsEnabled": true, - "Source": "Recommendations_raw", - "Query": "Recommendations_transform_v1_4()", - "IsTransactional": true, - "PropagateIngestionProperties": true -}] -``` - - -//===| Transactions |=================================================================================================== -// Supported versions: -// - MS CM EA reservation transactions: 2023-05-01 -- See https://learn.microsoft.com/azure/cost-management-billing/dataset-schema/reservation-transactions-ea -// - MS CM MCA reservation transactions: 2023-05-01 -- See https://learn.microsoft.com/azure/cost-management-billing/dataset-schema/reservation-transactions-mca -//====================================================================================================================== - -// Transactions_transform_v1_4 function -.create-or-alter function -with (docstring='All transactions transformed to FOCUS 1.4.', folder='Transactions') -Transactions_transform_v1_4() -{ - // NOTE: All open issues and questions are tracked @ https://github.com/microsoft/finops-toolkit/issues/1111 - let isoMonths = (duration: string) { - let number = toint(replace_regex(duration, @'[PMY]', '')); - toint(case( - duration == '', int(null), - duration endswith "Y", number * 12, - duration endswith "M", number, - -1 - )) - }; - Transactions_raw - // - // Set ProviderName - | extend ProviderName = 'Microsoft' - // - // Set source columns - | extend x_SourceName = coalesce(x_SourceName, iff(ProviderName == 'Microsoft', 'Cost Management', ProviderName)) - | extend x_SourceProvider = coalesce(x_SourceProvider, ProviderName) - | extend x_SourceType = coalesce(x_SourceType, iff(ProviderName == 'Microsoft', 'ReservationTransactions', '')) - | extend x_SourceVersion = coalesce(x_SourceVersion, iff(ProviderName == 'Microsoft', '2023-05-01', '')) - // - // Handle BillingPeriodStart/End - | extend BillingMonth = tostring(BillingMonth) - | extend BillingPeriodStart = iff(isempty(BillingMonth), datetime(null), todatetime(strcat(substring(BillingMonth, 0, 4), "-", substring(BillingMonth, 4, 2), "-", substring(BillingMonth, 6, 2)))) - | extend BillingPeriodEnd = iff(isempty(BillingMonth), datetime(null), startofmonth(endofmonth(BillingPeriodStart) + 1d)) - // - // Sort columns and apply final transforms - | project - BilledCost = Amount, - BillingAccountId = case( - BillingProfileId startswith '/', BillingProfileId, - isnotempty(CurrentEnrollmentId), strcat('/providers/Microsoft.Billing/billingAccounts/', CurrentEnrollmentId), - isnotempty(BillingProfileId), strcat('/providers/Microsoft.Billing/billingProfiles/', BillingProfileId), - '' - ), - BillingAccountName = coalesce(BillingProfileName, CurrentEnrollmentId), - BillingCurrency = Currency, - BillingPeriodEnd, - BillingPeriodStart, - ChargeCategory = case( - EventType in ('Cancel', 'Purchase', 'Refund'), 'Purchase', - 'Adjustment' - ), - ChargeClass = case( - EventType == 'Cancel', 'Cancel', // FOCUS does not handle this scenario - EventType == 'Refund', 'Correction', - '' - ), - ChargeDescription = Description, - ChargeFrequency = case( - BillingFrequency == 'OneTime', 'One-Time', - BillingFrequency == 'Recurring', 'Recurring', - BillingFrequency - ), - ChargePeriodStart = EventDate, - InvoiceId, - PricingQuantity = Quantity, - PricingUnit = 'Reservations', - ProviderName, - RegionId = Region, - RegionName = Region, - SubAccountId = iff(isempty(PurchasingSubscriptionGuid), '', strcat('/subscriptions/', PurchasingSubscriptionGuid)), - SubAccountName = iff(isempty(PurchasingSubscriptionGuid), '', PurchasingSubscriptionName), - x_AccountName = AccountName, - x_AccountOwnerId = AccountOwnerEmail, - x_CostCenter = CostCenter, - x_InvoiceNumber = Invoice, - x_InvoiceSectionId = InvoiceSectionId, - x_InvoiceSectionName = coalesce(InvoiceSectionName, DepartmentName), - x_IngestionTime = ingestion_time(), - x_MonetaryCommitment = MonetaryCommitment, - x_Overage = Overage, - x_PurchasingBillingAccountId = PurchasingEnrollment, - x_SkuOrderId = ReservationOrderId, - x_SkuOrderName = ReservationOrderName, - x_SkuSize = ArmSkuName, - x_SkuTerm = isoMonths(Term), - x_SourceName, - x_SourceProvider, - x_SourceType, - x_SourceVersion, - x_SubscriptionId = PurchasingSubscriptionGuid, - x_TransactionType = EventType -} - -// Transactions_final_v1_4 table -.create-merge table Transactions_final_v1_4 ( - BilledCost: real, // MS CM EA+MCA 2023-05-01 - BillingAccountId: string, // MS CM EA+MCA 2023-05-01 - BillingAccountName: string, // MS CM EA+MCA 2023-05-01 - BillingCurrency: string, // MS CM EA+MCA 2023-05-01 - BillingPeriodEnd: datetime, // MS CM EA+MCA 2023-05-01 - BillingPeriodStart: datetime, // MS CM EA+MCA 2023-05-01 - ChargeCategory: string, // Hubs add-on - ChargeClass: string, // Hubs add-on - ChargeDescription: string, // MS CM EA+MCA 2023-05-01 - ChargeFrequency: string, // MS CM EA+MCA 2023-05-01 - ChargePeriodStart: datetime, // MS CM EA+MCA 2023-05-01 - InvoiceId: string, // MS CM MCA 2023-05-01 - PricingQuantity: real, // MS CM EA+MCA 2023-05-01 - PricingUnit: string, // Hubs add-on - ProviderName: string, // Hubs add-on - RegionId: string, // MS CM EA+MCA 2023-05-01 - RegionName: string, // MS CM EA+MCA 2023-05-01 - SubAccountId: string, // MS CM EA+MCA 2023-05-01 - SubAccountName: string, // MS CM EA+MCA 2023-05-01 - x_AccountName: string, // MS CM EA 2023-05-01 - x_AccountOwnerId: string, // MS CM EA 2023-05-01 - x_CostCenter: string, // MS CM EA 2023-05-01 - x_InvoiceNumber: string, // MS CM MCA 2023-05-01 - x_InvoiceSectionId: string, // MS CM MCA 2023-05-01 - x_InvoiceSectionName: string, // MS CM MCA 2023-05-01 - x_IngestionTime: datetime, // Hubs add-on - x_MonetaryCommitment: real, // MS CM EA 2023-05-01 - x_Overage: real, // MS CM EA 2023-05-01 - x_PurchasingBillingAccountId: string, // MS CM EA 2023-05-01 - x_SkuOrderId: string, // MS CM EA+MCA 2023-05-01 - x_SkuOrderName: string, // MS CM EA+MCA 2023-05-01 - x_SkuSize: string, // MS CM EA+MCA 2023-05-01 - x_SkuTerm: int, // MS CM EA+MCA 2023-05-01 - x_SourceName: string, // Hubs add-on - x_SourceProvider: string, // Hubs add-on - x_SourceType: string, // Hubs add-on - x_SourceVersion: string, // Hubs add-on - x_SubscriptionId: string, // MS CM EA+MCA 2023-05-01 - x_TransactionType: string // MS CM EA+MCA 2023-05-01 -) - -// Update policy for Transactions_raw -> Transactions_final_v1_4 table -.alter table Transactions_final_v1_4 policy update -``` -[{ - "IsEnabled": true, - "Source": "Transactions_raw", - "Query": "Transactions_transform_v1_4()", - "IsTransactional": true, - "PropagateIngestionProperties": true -}] -``` diff --git a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_v1_4_Datasets.kql b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_v1_4_Datasets.kql new file mode 100644 index 000000000..432e95e72 --- /dev/null +++ b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_v1_4_Datasets.kql @@ -0,0 +1,669 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//====================================================================================================================== +// Ingestion database +// See known issues @ https://github.com/microsoft/finops-toolkit/issues/1111 +//====================================================================================================================== + +// For allowed commands, see https://learn.microsoft.com/azure/data-explorer/database-script + +// NOTE: FOCUS 1.4 ingestion setup is split into IngestionSetup_v1_4.kql (Prices + Costs) and IngestionSetup_v1_4_Datasets.kql +// (other datasets) to stay under the Bicep loadTextContent() 131,072-character limit. Scripts within a deployment batch +// are not guaranteed to run in order, so each file must remain self-contained (no cross-file references). + +//===| BillingPeriods |================================================================================================= +// Supported versions: +// - FOCUS 1.4+ (no Cost Management export shipped yet; defines the schema in advance for forward compat) +//====================================================================================================================== + +// BillingPeriods_transform_v1_4 function +.create-or-alter function +with (docstring='All billing periods aligned to FOCUS 1.4.', folder='BillingPeriods') +BillingPeriods_transform_v1_4() +{ + BillingPeriods_raw + | project + BillingPeriodCreated = column_ifexists('BillingPeriodCreated', datetime(null)), // FOCUS 1.4+ + BillingPeriodEnd, + BillingPeriodLastUpdated = column_ifexists('BillingPeriodLastUpdated', datetime(null)), // FOCUS 1.4+ + BillingPeriodStart, + BillingPeriodStatus = column_ifexists('BillingPeriodStatus', ''), // FOCUS 1.4+ + InvoiceIssuerName, + x_IngestionTime = ingestion_time(), + x_SourceName = coalesce(x_SourceName, ''), + x_SourceProvider = coalesce(x_SourceProvider, ''), + x_SourceType = coalesce(x_SourceType, ''), + x_SourceVersion = coalesce(x_SourceVersion, '') +} + +// BillingPeriods_final_v1_4 table +.create-merge table BillingPeriods_final_v1_4 ( + BillingPeriodCreated: datetime, // FOCUS 1.4+ + BillingPeriodEnd: datetime, // FOCUS 1.4+ + BillingPeriodLastUpdated: datetime, // FOCUS 1.4+ + BillingPeriodStart: datetime, // FOCUS 1.4+ + BillingPeriodStatus: string, // FOCUS 1.4+ + InvoiceIssuerName: string, // FOCUS 1.4+ + x_IngestionTime: datetime, // Hubs add-on + x_SourceName: string, // Hubs add-on + x_SourceProvider: string, // Hubs add-on + x_SourceType: string, // Hubs add-on + x_SourceVersion: string // Hubs add-on +) + +// Update policy for BillingPeriods_raw -> BillingPeriods_final_v1_4 table +.alter table BillingPeriods_final_v1_4 policy update +``` +[{ + "IsEnabled": true, + "Source": "BillingPeriods_raw", + "Query": "BillingPeriods_transform_v1_4()", + "IsTransactional": true, + "PropagateIngestionProperties": true +}] +``` + + +//===| CommitmentDiscountUsage |======================================================================================== +// Supported versions: +// - MS EA reservation details: 2023-03-01 -- See https://learn.microsoft.com/azure/cost-management-billing/dataset-schema/reservation-details-ea +// - MS MCA reservation details: 2023-03-01 -- See https://learn.microsoft.com/azure/cost-management-billing/dataset-schema/reservation-details-mca +//====================================================================================================================== + +// CommitmentDiscountUsage_transform_v1_4 function +.create-or-alter function +with (docstring='All commitment discount usage transformed to FOCUS 1.4. This includes reservationdeatils_raw.', folder='Commitment discounts') +CommitmentDiscountUsage_transform_v1_4() +{ + // NOTE: All open issues and questions are tracked @ https://github.com/microsoft/finops-toolkit/issues/1111 + CommitmentDiscountUsage_raw + // + // Set ProviderName + | extend ProviderName = 'Microsoft' + // + // Handle resource columns + | extend ResourceId = tolower(InstanceId) + | extend tmp_ResourceDetails = parse_resourceid(ResourceId) + | extend ResourceName = tostring(tmp_ResourceDetails.ResourceName) + | extend SubAccountId = tostring(tmp_ResourceDetails.SubAccountId) + | extend x_ResourceGroupName = tostring(tmp_ResourceDetails.x_ResourceGroupName) + | extend x_ResourceType = tostring(tmp_ResourceDetails.x_ResourceType) + | lookup kind=leftouter (ResourceTypes | distinct x_ResourceType, ResourceType = SingularDisplayName) on x_ResourceType + | lookup kind=leftouter (Services | distinct x_ResourceType, ServiceName, ServiceCategory, ServiceSubcategory, x_ServiceModel) on x_ResourceType + // + // Sort columns and apply final transforms + | project + ChargePeriodEnd = UsageDate + 1d, + ChargePeriodStart = UsageDate, + CommitmentDiscountCategory = 'Usage', + CommitmentDiscountId = tolower(strcat('/providers/microsoft.capacity/reservationorders/', ReservationOrderId, '/reservations/', ReservationId)), + CommitmentDiscountQuantity = UsedHours * InstanceFlexibilityRatio, + CommitmentDiscountType = 'Reservation', + CommitmentDiscountUnit = case( + InstanceFlexibilityRatio == 1, 'Hours', + InstanceFlexibilityRatio != 1, 'Normalized Hours', + '' + ), + ConsumedQuantity = UsedHours, + ProviderName, + ResourceId, + ResourceName, + ResourceType, + ServiceCategory, + ServiceName, + ServiceSubcategory, + SubAccountId, + x_CommitmentDiscountCommittedCount = TotalReservedQuantity, + x_CommitmentDiscountCommittedAmount = ReservedHours, + // TODO: Is this needed? -- x_CommitmentDiscountKind = Kind, + x_CommitmentDiscountNormalizedGroup = iff(InstanceFlexibilityGroup == 'NA', '', InstanceFlexibilityGroup), + x_CommitmentDiscountNormalizedRatio = InstanceFlexibilityRatio, + x_IngestionTime = ingestion_time(), + x_ResourceGroupName, + x_ResourceType, + // x_RowId = hash_sha256(strcat( + // // DO NOT CHANGE COLUMNS OR COLUMN ORDER + // CommitmentDiscountId, + // ResourceId, + // ChargePeriodStart + // )), + x_ServiceModel, + x_SkuOrderId = ReservationOrderId, + x_SkuSize = iff(SkuName == 'NA', '', SkuName), + x_SourceName = coalesce(x_SourceName, iff(ProviderName == 'Microsoft', 'Cost Management', ProviderName)), + x_SourceProvider = coalesce(x_SourceProvider, ProviderName), + x_SourceType = coalesce(x_SourceType, iff(ProviderName == 'Microsoft', 'ReservationDetails', '')), + x_SourceVersion = coalesce(x_SourceVersion, iff(ProviderName == 'Microsoft', '2024-03-01', '')) +} + +// CommitmentDiscountUsage_final_v1_4 table +.create-merge table CommitmentDiscountUsage_final_v1_4 ( + ChargePeriodEnd: datetime, // Hubs add-on + ChargePeriodStart: datetime, // MS 2023-03-01 + CommitmentDiscountCategory: string, // Hubs add-on + CommitmentDiscountId: string, // MS 2023-03-01 + CommitmentDiscountQuantity: real, // MS 2023-03-01 + CommitmentDiscountType: string, // Hubs add-on + CommitmentDiscountUnit: string, // Hubs add-on + ConsumedQuantity: real, // MS 2023-03-01 + ProviderName: string, // Hubs add-on + ResourceId: string, // MS 2023-03-01 + ResourceName: string, // Hubs add-on + ResourceType: string, // Hubs add-on + ServiceCategory: string, // Hubs add-on + ServiceName: string, // Hubs add-on + ServiceSubcategory: string, // Hubs add-on + SubAccountId: string, // Hubs add-on + x_CommitmentDiscountCommittedCount: real, // MS 2023-03-01 + x_CommitmentDiscountCommittedAmount: real, // MS 2023-03-01 + x_CommitmentDiscountNormalizedGroup: string, // MS 2023-03-01 + x_CommitmentDiscountNormalizedRatio: real, // MS 2023-03-01 + x_IngestionTime: datetime, // Hubs add-on + x_ResourceGroupName: string, // Hubs add-on + x_ResourceType: string, // Hubs add-on + x_ServiceModel: string, // Hubs add-on + x_SkuOrderId: string, // MS 2023-03-01 + x_SkuSize: string, // MS 2023-03-01 + x_SourceName: string, // Hubs add-on + x_SourceProvider: string, // Hubs add-on + x_SourceType: string, // Hubs add-on + x_SourceVersion: string // Hubs add-on +) + +// Update policy for CommitmentDiscountUsage_raw -> CommitmentDiscountUsage_final_v1_4 table +.alter table CommitmentDiscountUsage_final_v1_4 policy update +``` +[{ + "IsEnabled": true, + "Source": "CommitmentDiscountUsage_raw", + "Query": "CommitmentDiscountUsage_transform_v1_4()", + "IsTransactional": true, + "PropagateIngestionProperties": true +}] +``` + + +//===| Contract commitments |=========================================================================================== +// Supported versions: +// - FOCUS 1.4+ (no Cost Management export shipped yet; defines the schema in advance for forward compat) +//====================================================================================================================== + +// ContractCommitments_transform_v1_4 function +.create-or-alter function +with (docstring='All contract commitments aligned to FOCUS 1.4.', folder='Contract commitments') +ContractCommitments_transform_v1_4() +{ + ContractCommitments_raw + | project + BenefitCategory = column_ifexists('BenefitCategory', ''), // FOCUS 1.4+ + BillingCurrency, + ContractCommitmentApplicability = parse_json(column_ifexists('ContractCommitmentApplicability', '')),// FOCUS 1.4+ + ContractCommitmentCategory, + ContractCommitmentCost, + ContractCommitmentId, + ContractCommitmentPeriodEnd, + ContractCommitmentPeriodStart, + ContractCommitmentQuantity, + ContractCommitmentType, + ContractCommitmentUnit, + ContractId, + ContractPeriodEnd, + ContractPeriodStart, + Created = column_ifexists('Created', datetime(null)), // FOCUS 1.4+ + DiscountPercentage = column_ifexists('DiscountPercentage', real(null)), // FOCUS 1.4+ + DurationType = column_ifexists('DurationType', ''), // FOCUS 1.4+ + FulfillmentInterval = column_ifexists('FulfillmentInterval', ''), // FOCUS 1.4+ + InvoiceIssuerName, + LastUpdated = column_ifexists('LastUpdated', datetime(null)), // FOCUS 1.4+ + LifecycleStatus = column_ifexists('LifecycleStatus', ''), // FOCUS 1.4+ + Model = column_ifexists('Model', ''), // FOCUS 1.4+ + OfferCategory = column_ifexists('OfferCategory', ''), // FOCUS 1.4+ + PaymentInterval = column_ifexists('PaymentInterval', ''), // FOCUS 1.4+ + PaymentModel = column_ifexists('PaymentModel', ''), // FOCUS 1.4+ + PaymentUpfrontPercentage = column_ifexists('PaymentUpfrontPercentage', real(null)), // FOCUS 1.4+ + PricingCurrency, + PricingCurrencyContractCommitmentCost = column_ifexists('PricingCurrencyContractCommitmentCost', real(null)), // FOCUS 1.4+ + x_IngestionTime = ingestion_time(), + x_SourceName = coalesce(x_SourceName, ''), + x_SourceProvider = coalesce(x_SourceProvider, ''), + x_SourceType = coalesce(x_SourceType, ''), + x_SourceVersion = coalesce(x_SourceVersion, '') +} + +// ContractCommitments_final_v1_4 table +.create-merge table ContractCommitments_final_v1_4 ( + BenefitCategory: string, // FOCUS 1.4+ + BillingCurrency: string, // FOCUS 1.3+ + ContractCommitmentApplicability: dynamic, // FOCUS 1.4+ (JSON) + ContractCommitmentCategory: string, // FOCUS 1.3+ + ContractCommitmentCost: real, // FOCUS 1.3+ + ContractCommitmentId: string, // FOCUS 1.3+ + ContractCommitmentPeriodEnd: datetime, // FOCUS 1.3+ + ContractCommitmentPeriodStart: datetime, // FOCUS 1.3+ + ContractCommitmentQuantity: real, // FOCUS 1.3+ + ContractCommitmentType: string, // FOCUS 1.3+ + ContractCommitmentUnit: string, // FOCUS 1.3+ + ContractId: string, // FOCUS 1.3+ + ContractPeriodEnd: datetime, // FOCUS 1.3+ + ContractPeriodStart: datetime, // FOCUS 1.3+ + Created: datetime, // FOCUS 1.4+ + DiscountPercentage: real, // FOCUS 1.4+ + DurationType: string, // FOCUS 1.4+ + FulfillmentInterval: string, // FOCUS 1.4+ + InvoiceIssuerName: string, // FOCUS 1.3+ + LastUpdated: datetime, // FOCUS 1.4+ + LifecycleStatus: string, // FOCUS 1.4+ + Model: string, // FOCUS 1.4+ + OfferCategory: string, // FOCUS 1.4+ + PaymentInterval: string, // FOCUS 1.4+ + PaymentModel: string, // FOCUS 1.4+ + PaymentUpfrontPercentage: real, // FOCUS 1.4+ + PricingCurrency: string, // FOCUS 1.3+ + PricingCurrencyContractCommitmentCost: real, // FOCUS 1.4+ + x_IngestionTime: datetime, // Hubs add-on + x_SourceName: string, // Hubs add-on + x_SourceProvider: string, // Hubs add-on + x_SourceType: string, // Hubs add-on + x_SourceVersion: string // Hubs add-on +) + +// Update policy for ContractCommitments_raw -> ContractCommitments_final_v1_4 table +.alter table ContractCommitments_final_v1_4 policy update +``` +[{ + "IsEnabled": true, + "Source": "ContractCommitments_raw", + "Query": "ContractCommitments_transform_v1_4()", + "IsTransactional": true, + "PropagateIngestionProperties": true +}] +``` + + +//===| InvoiceDetails |================================================================================================= +// Supported versions: +// - FOCUS 1.4+ (no Cost Management export shipped yet; defines the schema in advance for forward compat) +//====================================================================================================================== + +// InvoiceDetails_transform_v1_4 function +.create-or-alter function +with (docstring='All invoice details aligned to FOCUS 1.4.', folder='InvoiceDetails') +InvoiceDetails_transform_v1_4() +{ + InvoiceDetails_raw + | project + BilledCost, + BillingAccountId, + BillingCurrency, + BillingPeriodEnd, + BillingPeriodStart, + ChargeCategory, + InvoiceDetailCreated = column_ifexists('InvoiceDetailCreated', datetime(null)), // FOCUS 1.4+ + InvoiceDetailDescription = column_ifexists('InvoiceDetailDescription', ''), // FOCUS 1.4+ + InvoiceDetailGrain = column_ifexists('InvoiceDetailGrain', ''), // FOCUS 1.4+ + InvoiceDetailId, + InvoiceDetailLastUpdated = column_ifexists('InvoiceDetailLastUpdated', datetime(null)), // FOCUS 1.4+ + InvoiceId, + InvoiceIssueDate = column_ifexists('InvoiceIssueDate', datetime(null)), // FOCUS 1.4+ + InvoiceIssueStatus = column_ifexists('InvoiceIssueStatus', ''), // FOCUS 1.4+ + InvoiceIssuerName, + PaymentCurrency = column_ifexists('PaymentCurrency', ''), // FOCUS 1.4+ + PaymentCurrencyBilledCost = column_ifexists('PaymentCurrencyBilledCost', real(null)), // FOCUS 1.4+ + PaymentCurrencyInvoiceDetailId = column_ifexists('PaymentCurrencyInvoiceDetailId', ''), // FOCUS 1.4+ + PaymentDueDate = column_ifexists('PaymentDueDate', datetime(null)), // FOCUS 1.4+ + PaymentTerms = column_ifexists('PaymentTerms', ''), // FOCUS 1.4+ + PurchaseOrderNumber = column_ifexists('PurchaseOrderNumber', ''), // FOCUS 1.4+ + ReferenceInvoiceId = column_ifexists('ReferenceInvoiceId', ''), // FOCUS 1.4+ + x_IngestionTime = ingestion_time(), + x_SourceName = coalesce(x_SourceName, ''), + x_SourceProvider = coalesce(x_SourceProvider, ''), + x_SourceType = coalesce(x_SourceType, ''), + x_SourceVersion = coalesce(x_SourceVersion, '') +} + +// InvoiceDetails_final_v1_4 table +.create-merge table InvoiceDetails_final_v1_4 ( + BilledCost: real, // FOCUS 1.4+ + BillingAccountId: string, // FOCUS 1.4+ + BillingCurrency: string, // FOCUS 1.4+ + BillingPeriodEnd: datetime, // FOCUS 1.4+ + BillingPeriodStart: datetime, // FOCUS 1.4+ + ChargeCategory: string, // FOCUS 1.4+ + InvoiceDetailCreated: datetime, // FOCUS 1.4+ + InvoiceDetailDescription: string, // FOCUS 1.4+ + InvoiceDetailGrain: string, // FOCUS 1.4+ + InvoiceDetailId: string, // FOCUS 1.4+ + InvoiceDetailLastUpdated: datetime, // FOCUS 1.4+ + InvoiceId: string, // FOCUS 1.4+ + InvoiceIssueDate: datetime, // FOCUS 1.4+ + InvoiceIssueStatus: string, // FOCUS 1.4+ + InvoiceIssuerName: string, // FOCUS 1.4+ + PaymentCurrency: string, // FOCUS 1.4+ + PaymentCurrencyBilledCost: real, // FOCUS 1.4+ + PaymentCurrencyInvoiceDetailId: string, // FOCUS 1.4+ + PaymentDueDate: datetime, // FOCUS 1.4+ + PaymentTerms: string, // FOCUS 1.4+ + PurchaseOrderNumber: string, // FOCUS 1.4+ + ReferenceInvoiceId: string, // FOCUS 1.4+ + x_IngestionTime: datetime, // Hubs add-on + x_SourceName: string, // Hubs add-on + x_SourceProvider: string, // Hubs add-on + x_SourceType: string, // Hubs add-on + x_SourceVersion: string // Hubs add-on +) + +// Update policy for InvoiceDetails_raw -> InvoiceDetails_final_v1_4 table +.alter table InvoiceDetails_final_v1_4 policy update +``` +[{ + "IsEnabled": true, + "Source": "InvoiceDetails_raw", + "Query": "InvoiceDetails_transform_v1_4()", + "IsTransactional": true, + "PropagateIngestionProperties": true +}] +``` + + +//===| Recommendations |================================================================================================ +// Supported datasets/versions: +// - MS CM EA reservation recommendations: 2023-05-01 -- See https://learn.microsoft.com/azure/cost-management-billing/dataset-schema/reservation-recommendations-ea +// - MS CM MCA reservation recommendations: 2023-05-01 -- See https://learn.microsoft.com/azure/cost-management-billing/dataset-schema/reservation-recommendations-mca +//====================================================================================================================== + +// Recommendations_transform_v1_4 function +.create-or-alter function +with (docstring='All recommendations transformed to FOCUS 1.4.', folder='Recommendations') +Recommendations_transform_v1_4() +{ + // NOTE: All open issues and questions are tracked @ https://github.com/microsoft/finops-toolkit/issues/1111 + let isoMonths = (duration: string) { + let number = toint(replace_regex(duration, @'[PMY]', '')); + toint(case( + duration == '', int(null), + duration endswith "Y", number * 12, + duration endswith "M", number, + -1 + )) + }; + Recommendations_raw + | extend x_IngestionTime = ingestion_time() + // + // Set ProviderName + | extend ProviderName = 'Microsoft' + // + // Set source columns + | extend x_SourceName = coalesce(x_SourceName, iff(ProviderName == 'Microsoft', 'Cost Management', ProviderName)) + | extend x_SourceProvider = coalesce(x_SourceProvider, ProviderName) + | extend x_SourceType = coalesce(x_SourceType, iff(ProviderName == 'Microsoft', 'ReservationRecommendations', '')) + | extend x_SourceVersion = coalesce(x_SourceVersion, iff(ProviderName == 'Microsoft', '2023-05-01', '')) + // + // Convert JSON cost columns to real + | extend CostWithNoReservedInstances = case(isnotempty(CostWithNoReservedInstances), CostWithNoReservedInstances, isnotempty(CostWithNoReservedInstancesJson), toreal(extract(@'"value":([0-9\.]+)', 1, CostWithNoReservedInstancesJson)), CostWithNoReservedInstances) + | extend NetSavings = case(isnotempty(NetSavings), NetSavings, isnotempty(NetSavingsJson), toreal(extract(@'"value":([0-9\.]+)', 1, NetSavingsJson)), NetSavings) + | extend TotalCostWithReservedInstances = case(isnotempty(TotalCostWithReservedInstances), TotalCostWithReservedInstances, isnotempty(TotalCostWithReservedInstancesJson), toreal(extract(@'"value":([0-9\.]+)', 1, TotalCostWithReservedInstancesJson)), TotalCostWithReservedInstances) + // + // Parse x_RecommendationDetails from JSON string (ARG queries serialize to string for Parquet compatibility) + | extend x_RecommendationDetails = iff(gettype(x_RecommendationDetails) == 'string', parse_json(tostring(x_RecommendationDetails)), x_RecommendationDetails) + // + // Normalize x_RecommendationDetails keys to x_PascalCase (Advisor extendedProperties use camelCase) + // Guard: inject a placeholder key so mv-apply doesn't drop rows with null/empty bags + | extend x_RecommendationDetails = bag_merge(coalesce(x_RecommendationDetails, dynamic({})), bag_pack('__placeholder', '')) + | mv-apply k = bag_keys(x_RecommendationDetails) on ( + where isnotempty(tostring(k)) and tostring(k) != '__placeholder' + | extend newKey = iff(tostring(k) startswith 'x_', tostring(k), strcat('x_', toupper(substring(tostring(k), 0, 1)), substring(tostring(k), 1))) + | summarize x_RecommendationDetails = make_bag(bag_pack(newKey, x_RecommendationDetails[tostring(k)])) + ) + // + // Build recommendation details + | lookup kind=leftouter (Regions | summarize RegionName = make_set(RegionName)[0] by Location = RegionId) on Location + | extend x_RecommendationDetails = case( + // Use incoming x_RecommendationDetails first + isnotempty(x_RecommendationDetails), x_RecommendationDetails, + // Create one for reservation recommendations if needed + x_SourceType == 'ReservationRecommendations', bag_pack( + 'CommitmentDiscountNormalizedGroup', InstanceFlexibilityGroup, + 'CommitmentDiscountNormalizedRatio', InstanceFlexibilityRatio, + 'CommitmentDiscountNormalizedSize', NormalizedSize, + 'CommitmentDiscountResourceType', ResourceType, + 'CommitmentDiscountScope', Scope, + 'LookbackPeriodDuration', case( + LookBackPeriod matches regex @'^Last([0-9]+)Days$', replace_regex(LookBackPeriod, @'^Last([0-9]+)Days$', @'P\1D'), + LookBackPeriod matches regex @'^[0-9]+$', strcat('P', LookBackPeriod, 'D'), + '' + ), + 'LookbackPeriodStart', FirstUsageDate, + 'RecommendedQuantity', RecommendedQuantity, + 'RecommendedQuantityNormalized', RecommendedQuantityNormalized, + 'RegionId', Location, + 'RegionName', RegionName, + 'SkuMeterId', MeterId, + 'SkuPriceDetails', SkuProperties, + 'SkuSize', coalesce(SKU, SkuName), + 'SkuTerm', isoMonths(Term) + ), + dynamic({}) + ) + // + // Prefer specified date, then fall back to generating a date based on reservation recommendation lookback period, then validate to ensure it's not in the future + | extend x_RecommendationDate = coalesce(x_RecommendationDate, FirstUsageDate + (toint(extract(@'^P([0-9]+)D$', 1, tostring(x_RecommendationDetails.LookbackPeriodDuration))) * 1d)) + | extend x_RecommendationDate = iff(x_RecommendationDate > now(), startofday(now()), x_RecommendationDate) + // + // Derive x_ResourceType from ResourceId + | extend tmp_ResourceType = tostring(parse_resourceid(ResourceId).x_ResourceType) + | extend x_RecommendationDetails = iff(isnotempty(tmp_ResourceType), bag_merge(bag_pack('x_ResourceType', tmp_ResourceType), x_RecommendationDetails), x_RecommendationDetails) + // + // Set ResourceType display name from x_ResourceType code + | extend ResourceType = coalesce(ResourceType, tostring(resource_type(tmp_ResourceType).SingularDisplayName), tmp_ResourceType) + // + | project + ProviderName, + ResourceId = tolower(ResourceId), // Force lowercase for consistent grouping/filtering + ResourceName = tolower(iff(tmp_ResourceType =~ 'microsoft.resources/subscriptions', coalesce(SubAccountName, ResourceName), ResourceName)), // Use subscription name for subscription-level recommendations + ResourceType, + SubAccountId = coalesce(SubAccountId, iff(isnotempty(SubscriptionId), strcat('/subscriptions/', SubscriptionId), '')), + SubAccountName, + x_EffectiveCostAfter = coalesce(x_EffectiveCostAfter, TotalCostWithReservedInstances), + x_EffectiveCostBefore = coalesce(x_EffectiveCostBefore, CostWithNoReservedInstances), + x_EffectiveCostSavings = coalesce(x_EffectiveCostSavings, NetSavings, toreal(x_RecommendationDetails.x_SavingsAmount), toreal(x_RecommendationDetails.x_AnnualSavingsAmount) / 12), + x_IngestionTime, + x_RecommendationCategory, // TODO: Set for reservation recommendations + x_RecommendationDate, + x_RecommendationDescription = coalesce(x_RecommendationDescription, tostring(x_RecommendationDetails.x_RecommendationSolution), tostring(x_RecommendationDetails.x_RecommendationSubCategory)), + x_RecommendationDetails, + x_RecommendationId = tolower(x_RecommendationId), // TODO: Set for reservation recommendations; force lowercase for consistent grouping/filtering + x_ResourceGroupName = tolower(x_ResourceGroupName), // Force lowercase for consistent grouping/filtering + x_SourceName, + x_SourceProvider, + x_SourceType, + x_SourceVersion +} + +// Recommendations_final_v1_4 table +.create-merge table Recommendations_final_v1_4 ( + ProviderName: string, + ResourceId: string, + ResourceName: string, + ResourceType: string, + SubAccountId: string, + SubAccountName: string, + x_EffectiveCostAfter: real, + x_EffectiveCostBefore: real, + x_EffectiveCostSavings: real, + x_IngestionTime: datetime, + x_RecommendationCategory: string, + x_RecommendationDate: datetime, + x_RecommendationDescription: string, + x_RecommendationDetails: dynamic, + x_RecommendationId: string, + x_ResourceGroupName: string, + x_SourceName: string, + x_SourceProvider: string, + x_SourceType: string, + x_SourceVersion: string +) + +// Update policy for Recommendations_raw -> Recommendations_final_v1_4 table +.alter table Recommendations_final_v1_4 policy update +``` +[{ + "IsEnabled": true, + "Source": "Recommendations_raw", + "Query": "Recommendations_transform_v1_4()", + "IsTransactional": true, + "PropagateIngestionProperties": true +}] +``` + + +//===| Transactions |=================================================================================================== +// Supported versions: +// - MS CM EA reservation transactions: 2023-05-01 -- See https://learn.microsoft.com/azure/cost-management-billing/dataset-schema/reservation-transactions-ea +// - MS CM MCA reservation transactions: 2023-05-01 -- See https://learn.microsoft.com/azure/cost-management-billing/dataset-schema/reservation-transactions-mca +//====================================================================================================================== + +// Transactions_transform_v1_4 function +.create-or-alter function +with (docstring='All transactions transformed to FOCUS 1.4.', folder='Transactions') +Transactions_transform_v1_4() +{ + // NOTE: All open issues and questions are tracked @ https://github.com/microsoft/finops-toolkit/issues/1111 + let isoMonths = (duration: string) { + let number = toint(replace_regex(duration, @'[PMY]', '')); + toint(case( + duration == '', int(null), + duration endswith "Y", number * 12, + duration endswith "M", number, + -1 + )) + }; + Transactions_raw + // + // Set ProviderName + | extend ProviderName = 'Microsoft' + // + // Set source columns + | extend x_SourceName = coalesce(x_SourceName, iff(ProviderName == 'Microsoft', 'Cost Management', ProviderName)) + | extend x_SourceProvider = coalesce(x_SourceProvider, ProviderName) + | extend x_SourceType = coalesce(x_SourceType, iff(ProviderName == 'Microsoft', 'ReservationTransactions', '')) + | extend x_SourceVersion = coalesce(x_SourceVersion, iff(ProviderName == 'Microsoft', '2023-05-01', '')) + // + // Handle BillingPeriodStart/End + | extend BillingMonth = tostring(BillingMonth) + | extend BillingPeriodStart = iff(isempty(BillingMonth), datetime(null), todatetime(strcat(substring(BillingMonth, 0, 4), "-", substring(BillingMonth, 4, 2), "-", substring(BillingMonth, 6, 2)))) + | extend BillingPeriodEnd = iff(isempty(BillingMonth), datetime(null), startofmonth(endofmonth(BillingPeriodStart) + 1d)) + // + // Sort columns and apply final transforms + | project + BilledCost = Amount, + BillingAccountId = case( + BillingProfileId startswith '/', BillingProfileId, + isnotempty(CurrentEnrollmentId), strcat('/providers/Microsoft.Billing/billingAccounts/', CurrentEnrollmentId), + isnotempty(BillingProfileId), strcat('/providers/Microsoft.Billing/billingProfiles/', BillingProfileId), + '' + ), + BillingAccountName = coalesce(BillingProfileName, CurrentEnrollmentId), + BillingCurrency = Currency, + BillingPeriodEnd, + BillingPeriodStart, + ChargeCategory = case( + EventType in ('Cancel', 'Purchase', 'Refund'), 'Purchase', + 'Adjustment' + ), + ChargeClass = case( + EventType == 'Cancel', 'Cancel', // FOCUS does not handle this scenario + EventType == 'Refund', 'Correction', + '' + ), + ChargeDescription = Description, + ChargeFrequency = case( + BillingFrequency == 'OneTime', 'One-Time', + BillingFrequency == 'Recurring', 'Recurring', + BillingFrequency + ), + ChargePeriodStart = EventDate, + InvoiceId, + PricingQuantity = Quantity, + PricingUnit = 'Reservations', + ProviderName, + RegionId = Region, + RegionName = Region, + SubAccountId = iff(isempty(PurchasingSubscriptionGuid), '', strcat('/subscriptions/', PurchasingSubscriptionGuid)), + SubAccountName = iff(isempty(PurchasingSubscriptionGuid), '', PurchasingSubscriptionName), + x_AccountName = AccountName, + x_AccountOwnerId = AccountOwnerEmail, + x_CostCenter = CostCenter, + x_InvoiceNumber = Invoice, + x_InvoiceSectionId = InvoiceSectionId, + x_InvoiceSectionName = coalesce(InvoiceSectionName, DepartmentName), + x_IngestionTime = ingestion_time(), + x_MonetaryCommitment = MonetaryCommitment, + x_Overage = Overage, + x_PurchasingBillingAccountId = PurchasingEnrollment, + x_SkuOrderId = ReservationOrderId, + x_SkuOrderName = ReservationOrderName, + x_SkuSize = ArmSkuName, + x_SkuTerm = isoMonths(Term), + x_SourceName, + x_SourceProvider, + x_SourceType, + x_SourceVersion, + x_SubscriptionId = PurchasingSubscriptionGuid, + x_TransactionType = EventType +} + +// Transactions_final_v1_4 table +.create-merge table Transactions_final_v1_4 ( + BilledCost: real, // MS CM EA+MCA 2023-05-01 + BillingAccountId: string, // MS CM EA+MCA 2023-05-01 + BillingAccountName: string, // MS CM EA+MCA 2023-05-01 + BillingCurrency: string, // MS CM EA+MCA 2023-05-01 + BillingPeriodEnd: datetime, // MS CM EA+MCA 2023-05-01 + BillingPeriodStart: datetime, // MS CM EA+MCA 2023-05-01 + ChargeCategory: string, // Hubs add-on + ChargeClass: string, // Hubs add-on + ChargeDescription: string, // MS CM EA+MCA 2023-05-01 + ChargeFrequency: string, // MS CM EA+MCA 2023-05-01 + ChargePeriodStart: datetime, // MS CM EA+MCA 2023-05-01 + InvoiceId: string, // MS CM MCA 2023-05-01 + PricingQuantity: real, // MS CM EA+MCA 2023-05-01 + PricingUnit: string, // Hubs add-on + ProviderName: string, // Hubs add-on + RegionId: string, // MS CM EA+MCA 2023-05-01 + RegionName: string, // MS CM EA+MCA 2023-05-01 + SubAccountId: string, // MS CM EA+MCA 2023-05-01 + SubAccountName: string, // MS CM EA+MCA 2023-05-01 + x_AccountName: string, // MS CM EA 2023-05-01 + x_AccountOwnerId: string, // MS CM EA 2023-05-01 + x_CostCenter: string, // MS CM EA 2023-05-01 + x_InvoiceNumber: string, // MS CM MCA 2023-05-01 + x_InvoiceSectionId: string, // MS CM MCA 2023-05-01 + x_InvoiceSectionName: string, // MS CM MCA 2023-05-01 + x_IngestionTime: datetime, // Hubs add-on + x_MonetaryCommitment: real, // MS CM EA 2023-05-01 + x_Overage: real, // MS CM EA 2023-05-01 + x_PurchasingBillingAccountId: string, // MS CM EA 2023-05-01 + x_SkuOrderId: string, // MS CM EA+MCA 2023-05-01 + x_SkuOrderName: string, // MS CM EA+MCA 2023-05-01 + x_SkuSize: string, // MS CM EA+MCA 2023-05-01 + x_SkuTerm: int, // MS CM EA+MCA 2023-05-01 + x_SourceName: string, // Hubs add-on + x_SourceProvider: string, // Hubs add-on + x_SourceType: string, // Hubs add-on + x_SourceVersion: string, // Hubs add-on + x_SubscriptionId: string, // MS CM EA+MCA 2023-05-01 + x_TransactionType: string // MS CM EA+MCA 2023-05-01 +) + +// Update policy for Transactions_raw -> Transactions_final_v1_4 table +.alter table Transactions_final_v1_4 policy update +``` +[{ + "IsEnabled": true, + "Source": "Transactions_raw", + "Query": "Transactions_transform_v1_4()", + "IsTransactional": true, + "PropagateIngestionProperties": true +}] +``` From f24f0d82ba7f5cf948068f46dc851c28eb85a4d3 Mon Sep 17 00:00:00 2001 From: Michael Flanakin Date: Wed, 15 Jul 2026 16:57:45 -0700 Subject: [PATCH 5/6] fix: port dev v1_2 KQL fixes into FOCUS 1.4 ingestion transforms Ports #2151 (strcat-free emptiness checks), #2189 (max()-based price lookup for duplicate on-demand rows), and the AmortizedCost docstring copy-paste fix into IngestionSetup_v1_4.kql, which was copied from v1_2 before those fixes landed on dev. Co-Authored-By: Claude Fable 5 --- .../Analytics/scripts/IngestionSetup_v1_4.kql | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_v1_4.kql b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_v1_4.kql index 88b5f14e5..6b1d6e826 100644 --- a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_v1_4.kql +++ b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_v1_4.kql @@ -418,7 +418,7 @@ Costs_transform_v1_4() and (isempty(ListUnitPrice) or isempty(ContractedUnitPrice) or ListUnitPrice == 0 or ContractedUnitPrice == 0) and x_EffectiveUnitPrice != 0 and not(CommitmentDiscountCategory == 'Spend' and CommitmentDiscountStatus == 'Unused') - and isnotempty(strcat(x_SkuMeterId, x_SkuOfferId)) + and isnotempty(x_SkuMeterId) and isnotempty(x_SkuOfferId) | as allCosts | where tmp_MissingPrices | extend tmp_ReservationPriceLookupKey = tolower(strcat(x_BillingProfileId, substring(ChargePeriodStart, 0, 7), x_SkuMeterId, x_SkuOfferId)) @@ -427,7 +427,8 @@ Costs_transform_v1_4() Prices_final_v1_4 | extend tmp_ReservationPriceLookupKey = tolower(strcat(x_BillingProfileId, substring(x_EffectivePeriodStart, 0, 7), x_SkuMeterId, x_SkuOfferId)) | where x_SkuPriceType == 'Consumption' and tmp_ReservationPriceLookupKey in ((costsWithMissingPrices | summarize by tmp_ReservationPriceLookupKey)) - | summarize ListUnitPrice = min(ListUnitPrice), ContractedUnitPrice = min(ContractedUnitPrice) by tmp_ReservationPriceLookupKey, x_PricingBlockSize, PricingUnit + // When duplicate price rows collapse under one lookup key, use the highest on-demand price as the savings baseline; min() can pick an anomalously low row and underreport savings + | summarize ListUnitPrice = max(ListUnitPrice), ContractedUnitPrice = max(ContractedUnitPrice) by tmp_ReservationPriceLookupKey, x_PricingBlockSize, PricingUnit ) on tmp_ReservationPriceLookupKey // // Select the best price to use for each row @@ -1390,7 +1391,7 @@ ActualCosts_transform_v1_4() // AmortizedCosts_transform_v1_4 function .create-or-alter function -with (docstring='ActualCost exports transformed to FOCUS 1.4.', folder='Costs') +with (docstring='AmortizedCost exports transformed to FOCUS 1.4.', folder='Costs') AmortizedCosts_transform_v1_4() { // NOTE: All open issues and questions are tracked @ https://github.com/microsoft/finops-toolkit/issues/1111 From e7f61f2b28bdc4280247d876498be4e6e0630008 Mon Sep 17 00:00:00 2001 From: Michael Flanakin Date: Wed, 15 Jul 2026 17:03:49 -0700 Subject: [PATCH 6/6] fix: align ContractCommitments schema with FOCUS 1.4 spec column IDs Renames 12 unprefixed columns (BenefitCategory, Created, Model, ...) to their spec IDs (ContractCommitmentBenefitCategory, ...) and adds the two missing columns (ContractCommitmentDescription, ServiceProviderName), bringing the dataset to the full 30 FOCUS columns across the raw table, parquet mapping, transform, and final table. Hub x_ add-on columns are unchanged. Co-Authored-By: Claude Fable 5 --- .../scripts/IngestionSetup_RawTables.kql | 132 +++++++++--------- .../scripts/IngestionSetup_v1_4_Datasets.kql | 98 ++++++------- 2 files changed, 119 insertions(+), 111 deletions(-) diff --git a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_RawTables.kql b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_RawTables.kql index f015577f6..d3ef9e27c 100644 --- a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_RawTables.kql +++ b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_RawTables.kql @@ -400,76 +400,80 @@ // ContractCommitments_raw table -- Redefine all columns .alter table ContractCommitments_raw ( - BenefitCategory: string, // FOCUS 1.4+ - BillingCurrency: string, // FOCUS 1.3+ - ContractCommitmentApplicability: string, // FOCUS 1.4+ (JSON) - ContractCommitmentCategory: string, // FOCUS 1.3+ - ContractCommitmentCost: real, // FOCUS 1.3+ - ContractCommitmentId: string, // FOCUS 1.3+ - ContractCommitmentPeriodEnd: datetime, // FOCUS 1.3+ - ContractCommitmentPeriodStart: datetime, // FOCUS 1.3+ - ContractCommitmentQuantity: real, // FOCUS 1.3+ - ContractCommitmentType: string, // FOCUS 1.3+ - ContractCommitmentUnit: string, // FOCUS 1.3+ - ContractId: string, // FOCUS 1.3+ - ContractPeriodEnd: datetime, // FOCUS 1.3+ - ContractPeriodStart: datetime, // FOCUS 1.3+ - Created: datetime, // FOCUS 1.4+ - DiscountPercentage: real, // FOCUS 1.4+ - DurationType: string, // FOCUS 1.4+ - FulfillmentInterval: string, // FOCUS 1.4+ - InvoiceIssuerName: string, // FOCUS 1.3+ - LastUpdated: datetime, // FOCUS 1.4+ - LifecycleStatus: string, // FOCUS 1.4+ - Model: string, // FOCUS 1.4+ - OfferCategory: string, // FOCUS 1.4+ - PaymentInterval: string, // FOCUS 1.4+ - PaymentModel: string, // FOCUS 1.4+ - PaymentUpfrontPercentage: real, // FOCUS 1.4+ - PricingCurrency: string, // FOCUS 1.3+ - PricingCurrencyContractCommitmentCost: real, // FOCUS 1.4+ - x_SourceName: string, // Hubs v1_4+ - x_SourceProvider: string, // Hubs v1_4+ - x_SourceType: string, // Hubs v1_4+ - x_SourceVersion: string // Hubs v1_4+ + BillingCurrency: string, // FOCUS 1.3+ + ContractCommitmentApplicability: string, // FOCUS 1.4+ (JSON) + ContractCommitmentBenefitCategory: string, // FOCUS 1.4+ + ContractCommitmentCategory: string, // FOCUS 1.3+ + ContractCommitmentCost: real, // FOCUS 1.3+ + ContractCommitmentCreated: datetime, // FOCUS 1.4+ + ContractCommitmentDescription: string, // FOCUS 1.3+ + ContractCommitmentDiscountPercentage: real, // FOCUS 1.4+ + ContractCommitmentDurationType: string, // FOCUS 1.4+ + ContractCommitmentFulfillmentInterval: string, // FOCUS 1.4+ + ContractCommitmentId: string, // FOCUS 1.3+ + ContractCommitmentLastUpdated: datetime, // FOCUS 1.4+ + ContractCommitmentLifecycleStatus: string, // FOCUS 1.4+ + ContractCommitmentModel: string, // FOCUS 1.4+ + ContractCommitmentOfferCategory: string, // FOCUS 1.4+ + ContractCommitmentPaymentInterval: string, // FOCUS 1.4+ + ContractCommitmentPaymentModel: string, // FOCUS 1.4+ + ContractCommitmentPaymentUpfrontPercentage: real, // FOCUS 1.4+ + ContractCommitmentPeriodEnd: datetime, // FOCUS 1.3+ + ContractCommitmentPeriodStart: datetime, // FOCUS 1.3+ + ContractCommitmentQuantity: real, // FOCUS 1.3+ + ContractCommitmentType: string, // FOCUS 1.3+ + ContractCommitmentUnit: string, // FOCUS 1.3+ + ContractId: string, // FOCUS 1.3+ + ContractPeriodEnd: datetime, // FOCUS 1.3+ + ContractPeriodStart: datetime, // FOCUS 1.3+ + InvoiceIssuerName: string, // FOCUS 1.3+ + PricingCurrency: string, // FOCUS 1.3+ + PricingCurrencyContractCommitmentCost: real, // FOCUS 1.4+ + ServiceProviderName: string, // FOCUS 1.4+ + x_SourceName: string, // Hubs v1_4+ + x_SourceProvider: string, // Hubs v1_4+ + x_SourceType: string, // Hubs v1_4+ + x_SourceVersion: string // Hubs v1_4+ ) // ContractCommitments_raw ingestion mapping .create-or-alter table ContractCommitments_raw ingestion parquet mapping "ContractCommitments_raw_mapping" ``` [ - { "Column": "BenefitCategory", "Properties": { "Field": "BenefitCategory" } }, - { "Column": "BillingCurrency", "Properties": { "Field": "BillingCurrency" } }, - { "Column": "ContractCommitmentApplicability", "Properties": { "Field": "ContractCommitmentApplicability" } }, - { "Column": "ContractCommitmentCategory", "Properties": { "Field": "ContractCommitmentCategory" } }, - { "Column": "ContractCommitmentCost", "Properties": { "Field": "ContractCommitmentCost" } }, - { "Column": "ContractCommitmentId", "Properties": { "Field": "ContractCommitmentId" } }, - { "Column": "ContractCommitmentPeriodEnd", "Properties": { "Field": "ContractCommitmentPeriodEnd" } }, - { "Column": "ContractCommitmentPeriodStart", "Properties": { "Field": "ContractCommitmentPeriodStart" } }, - { "Column": "ContractCommitmentQuantity", "Properties": { "Field": "ContractCommitmentQuantity" } }, - { "Column": "ContractCommitmentType", "Properties": { "Field": "ContractCommitmentType" } }, - { "Column": "ContractCommitmentUnit", "Properties": { "Field": "ContractCommitmentUnit" } }, - { "Column": "ContractId", "Properties": { "Field": "ContractId" } }, - { "Column": "ContractPeriodEnd", "Properties": { "Field": "ContractPeriodEnd" } }, - { "Column": "ContractPeriodStart", "Properties": { "Field": "ContractPeriodStart" } }, - { "Column": "Created", "Properties": { "Field": "Created" } }, - { "Column": "DiscountPercentage", "Properties": { "Field": "DiscountPercentage" } }, - { "Column": "DurationType", "Properties": { "Field": "DurationType" } }, - { "Column": "FulfillmentInterval", "Properties": { "Field": "FulfillmentInterval" } }, - { "Column": "InvoiceIssuerName", "Properties": { "Field": "InvoiceIssuerName" } }, - { "Column": "LastUpdated", "Properties": { "Field": "LastUpdated" } }, - { "Column": "LifecycleStatus", "Properties": { "Field": "LifecycleStatus" } }, - { "Column": "Model", "Properties": { "Field": "Model" } }, - { "Column": "OfferCategory", "Properties": { "Field": "OfferCategory" } }, - { "Column": "PaymentInterval", "Properties": { "Field": "PaymentInterval" } }, - { "Column": "PaymentModel", "Properties": { "Field": "PaymentModel" } }, - { "Column": "PaymentUpfrontPercentage", "Properties": { "Field": "PaymentUpfrontPercentage" } }, - { "Column": "PricingCurrency", "Properties": { "Field": "PricingCurrency" } }, - { "Column": "PricingCurrencyContractCommitmentCost", "Properties": { "Field": "PricingCurrencyContractCommitmentCost" } }, - { "Column": "x_SourceName", "Properties": { "Field": "x_SourceName" } }, - { "Column": "x_SourceProvider", "Properties": { "Field": "x_SourceProvider" } }, - { "Column": "x_SourceType", "Properties": { "Field": "x_SourceType" } }, - { "Column": "x_SourceVersion", "Properties": { "Field": "x_SourceVersion" } } + { "Column": "BillingCurrency", "Properties": { "Field": "BillingCurrency" } }, + { "Column": "ContractCommitmentApplicability", "Properties": { "Field": "ContractCommitmentApplicability" } }, + { "Column": "ContractCommitmentBenefitCategory", "Properties": { "Field": "ContractCommitmentBenefitCategory" } }, + { "Column": "ContractCommitmentCategory", "Properties": { "Field": "ContractCommitmentCategory" } }, + { "Column": "ContractCommitmentCost", "Properties": { "Field": "ContractCommitmentCost" } }, + { "Column": "ContractCommitmentCreated", "Properties": { "Field": "ContractCommitmentCreated" } }, + { "Column": "ContractCommitmentDescription", "Properties": { "Field": "ContractCommitmentDescription" } }, + { "Column": "ContractCommitmentDiscountPercentage", "Properties": { "Field": "ContractCommitmentDiscountPercentage" } }, + { "Column": "ContractCommitmentDurationType", "Properties": { "Field": "ContractCommitmentDurationType" } }, + { "Column": "ContractCommitmentFulfillmentInterval", "Properties": { "Field": "ContractCommitmentFulfillmentInterval" } }, + { "Column": "ContractCommitmentId", "Properties": { "Field": "ContractCommitmentId" } }, + { "Column": "ContractCommitmentLastUpdated", "Properties": { "Field": "ContractCommitmentLastUpdated" } }, + { "Column": "ContractCommitmentLifecycleStatus", "Properties": { "Field": "ContractCommitmentLifecycleStatus" } }, + { "Column": "ContractCommitmentModel", "Properties": { "Field": "ContractCommitmentModel" } }, + { "Column": "ContractCommitmentOfferCategory", "Properties": { "Field": "ContractCommitmentOfferCategory" } }, + { "Column": "ContractCommitmentPaymentInterval", "Properties": { "Field": "ContractCommitmentPaymentInterval" } }, + { "Column": "ContractCommitmentPaymentModel", "Properties": { "Field": "ContractCommitmentPaymentModel" } }, + { "Column": "ContractCommitmentPaymentUpfrontPercentage", "Properties": { "Field": "ContractCommitmentPaymentUpfrontPercentage" } }, + { "Column": "ContractCommitmentPeriodEnd", "Properties": { "Field": "ContractCommitmentPeriodEnd" } }, + { "Column": "ContractCommitmentPeriodStart", "Properties": { "Field": "ContractCommitmentPeriodStart" } }, + { "Column": "ContractCommitmentQuantity", "Properties": { "Field": "ContractCommitmentQuantity" } }, + { "Column": "ContractCommitmentType", "Properties": { "Field": "ContractCommitmentType" } }, + { "Column": "ContractCommitmentUnit", "Properties": { "Field": "ContractCommitmentUnit" } }, + { "Column": "ContractId", "Properties": { "Field": "ContractId" } }, + { "Column": "ContractPeriodEnd", "Properties": { "Field": "ContractPeriodEnd" } }, + { "Column": "ContractPeriodStart", "Properties": { "Field": "ContractPeriodStart" } }, + { "Column": "InvoiceIssuerName", "Properties": { "Field": "InvoiceIssuerName" } }, + { "Column": "PricingCurrency", "Properties": { "Field": "PricingCurrency" } }, + { "Column": "PricingCurrencyContractCommitmentCost", "Properties": { "Field": "PricingCurrencyContractCommitmentCost" } }, + { "Column": "ServiceProviderName", "Properties": { "Field": "ServiceProviderName" } }, + { "Column": "x_SourceName", "Properties": { "Field": "x_SourceName" } }, + { "Column": "x_SourceProvider", "Properties": { "Field": "x_SourceProvider" } }, + { "Column": "x_SourceType", "Properties": { "Field": "x_SourceType" } }, + { "Column": "x_SourceVersion", "Properties": { "Field": "x_SourceVersion" } } ] ``` diff --git a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_v1_4_Datasets.kql b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_v1_4_Datasets.kql index 432e95e72..458e110ca 100644 --- a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_v1_4_Datasets.kql +++ b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_v1_4_Datasets.kql @@ -196,12 +196,24 @@ ContractCommitments_transform_v1_4() { ContractCommitments_raw | project - BenefitCategory = column_ifexists('BenefitCategory', ''), // FOCUS 1.4+ BillingCurrency, - ContractCommitmentApplicability = parse_json(column_ifexists('ContractCommitmentApplicability', '')),// FOCUS 1.4+ + ContractCommitmentApplicability = parse_json(column_ifexists('ContractCommitmentApplicability', '')), // FOCUS 1.4+ + ContractCommitmentBenefitCategory = column_ifexists('ContractCommitmentBenefitCategory', ''), // FOCUS 1.4+ ContractCommitmentCategory, ContractCommitmentCost, + ContractCommitmentCreated = column_ifexists('ContractCommitmentCreated', datetime(null)), // FOCUS 1.4+ + ContractCommitmentDescription = column_ifexists('ContractCommitmentDescription', ''), // FOCUS 1.3+ + ContractCommitmentDiscountPercentage = column_ifexists('ContractCommitmentDiscountPercentage', real(null)), // FOCUS 1.4+ + ContractCommitmentDurationType = column_ifexists('ContractCommitmentDurationType', ''), // FOCUS 1.4+ + ContractCommitmentFulfillmentInterval = column_ifexists('ContractCommitmentFulfillmentInterval', ''), // FOCUS 1.4+ ContractCommitmentId, + ContractCommitmentLastUpdated = column_ifexists('ContractCommitmentLastUpdated', datetime(null)), // FOCUS 1.4+ + ContractCommitmentLifecycleStatus = column_ifexists('ContractCommitmentLifecycleStatus', ''), // FOCUS 1.4+ + ContractCommitmentModel = column_ifexists('ContractCommitmentModel', ''), // FOCUS 1.4+ + ContractCommitmentOfferCategory = column_ifexists('ContractCommitmentOfferCategory', ''), // FOCUS 1.4+ + ContractCommitmentPaymentInterval = column_ifexists('ContractCommitmentPaymentInterval', ''), // FOCUS 1.4+ + ContractCommitmentPaymentModel = column_ifexists('ContractCommitmentPaymentModel', ''), // FOCUS 1.4+ + ContractCommitmentPaymentUpfrontPercentage = column_ifexists('ContractCommitmentPaymentUpfrontPercentage', real(null)), // FOCUS 1.4+ ContractCommitmentPeriodEnd, ContractCommitmentPeriodStart, ContractCommitmentQuantity, @@ -210,20 +222,10 @@ ContractCommitments_transform_v1_4() ContractId, ContractPeriodEnd, ContractPeriodStart, - Created = column_ifexists('Created', datetime(null)), // FOCUS 1.4+ - DiscountPercentage = column_ifexists('DiscountPercentage', real(null)), // FOCUS 1.4+ - DurationType = column_ifexists('DurationType', ''), // FOCUS 1.4+ - FulfillmentInterval = column_ifexists('FulfillmentInterval', ''), // FOCUS 1.4+ InvoiceIssuerName, - LastUpdated = column_ifexists('LastUpdated', datetime(null)), // FOCUS 1.4+ - LifecycleStatus = column_ifexists('LifecycleStatus', ''), // FOCUS 1.4+ - Model = column_ifexists('Model', ''), // FOCUS 1.4+ - OfferCategory = column_ifexists('OfferCategory', ''), // FOCUS 1.4+ - PaymentInterval = column_ifexists('PaymentInterval', ''), // FOCUS 1.4+ - PaymentModel = column_ifexists('PaymentModel', ''), // FOCUS 1.4+ - PaymentUpfrontPercentage = column_ifexists('PaymentUpfrontPercentage', real(null)), // FOCUS 1.4+ PricingCurrency, - PricingCurrencyContractCommitmentCost = column_ifexists('PricingCurrencyContractCommitmentCost', real(null)), // FOCUS 1.4+ + PricingCurrencyContractCommitmentCost = column_ifexists('PricingCurrencyContractCommitmentCost', real(null)), // FOCUS 1.4+ + ServiceProviderName = column_ifexists('ServiceProviderName', ''), // FOCUS 1.4+ x_IngestionTime = ingestion_time(), x_SourceName = coalesce(x_SourceName, ''), x_SourceProvider = coalesce(x_SourceProvider, ''), @@ -233,39 +235,41 @@ ContractCommitments_transform_v1_4() // ContractCommitments_final_v1_4 table .create-merge table ContractCommitments_final_v1_4 ( - BenefitCategory: string, // FOCUS 1.4+ - BillingCurrency: string, // FOCUS 1.3+ - ContractCommitmentApplicability: dynamic, // FOCUS 1.4+ (JSON) - ContractCommitmentCategory: string, // FOCUS 1.3+ - ContractCommitmentCost: real, // FOCUS 1.3+ - ContractCommitmentId: string, // FOCUS 1.3+ - ContractCommitmentPeriodEnd: datetime, // FOCUS 1.3+ - ContractCommitmentPeriodStart: datetime, // FOCUS 1.3+ - ContractCommitmentQuantity: real, // FOCUS 1.3+ - ContractCommitmentType: string, // FOCUS 1.3+ - ContractCommitmentUnit: string, // FOCUS 1.3+ - ContractId: string, // FOCUS 1.3+ - ContractPeriodEnd: datetime, // FOCUS 1.3+ - ContractPeriodStart: datetime, // FOCUS 1.3+ - Created: datetime, // FOCUS 1.4+ - DiscountPercentage: real, // FOCUS 1.4+ - DurationType: string, // FOCUS 1.4+ - FulfillmentInterval: string, // FOCUS 1.4+ - InvoiceIssuerName: string, // FOCUS 1.3+ - LastUpdated: datetime, // FOCUS 1.4+ - LifecycleStatus: string, // FOCUS 1.4+ - Model: string, // FOCUS 1.4+ - OfferCategory: string, // FOCUS 1.4+ - PaymentInterval: string, // FOCUS 1.4+ - PaymentModel: string, // FOCUS 1.4+ - PaymentUpfrontPercentage: real, // FOCUS 1.4+ - PricingCurrency: string, // FOCUS 1.3+ - PricingCurrencyContractCommitmentCost: real, // FOCUS 1.4+ - x_IngestionTime: datetime, // Hubs add-on - x_SourceName: string, // Hubs add-on - x_SourceProvider: string, // Hubs add-on - x_SourceType: string, // Hubs add-on - x_SourceVersion: string // Hubs add-on + BillingCurrency: string, // FOCUS 1.3+ + ContractCommitmentApplicability: dynamic, // FOCUS 1.4+ (JSON) + ContractCommitmentBenefitCategory: string, // FOCUS 1.4+ + ContractCommitmentCategory: string, // FOCUS 1.3+ + ContractCommitmentCost: real, // FOCUS 1.3+ + ContractCommitmentCreated: datetime, // FOCUS 1.4+ + ContractCommitmentDescription: string, // FOCUS 1.3+ + ContractCommitmentDiscountPercentage: real, // FOCUS 1.4+ + ContractCommitmentDurationType: string, // FOCUS 1.4+ + ContractCommitmentFulfillmentInterval: string, // FOCUS 1.4+ + ContractCommitmentId: string, // FOCUS 1.3+ + ContractCommitmentLastUpdated: datetime, // FOCUS 1.4+ + ContractCommitmentLifecycleStatus: string, // FOCUS 1.4+ + ContractCommitmentModel: string, // FOCUS 1.4+ + ContractCommitmentOfferCategory: string, // FOCUS 1.4+ + ContractCommitmentPaymentInterval: string, // FOCUS 1.4+ + ContractCommitmentPaymentModel: string, // FOCUS 1.4+ + ContractCommitmentPaymentUpfrontPercentage: real, // FOCUS 1.4+ + ContractCommitmentPeriodEnd: datetime, // FOCUS 1.3+ + ContractCommitmentPeriodStart: datetime, // FOCUS 1.3+ + ContractCommitmentQuantity: real, // FOCUS 1.3+ + ContractCommitmentType: string, // FOCUS 1.3+ + ContractCommitmentUnit: string, // FOCUS 1.3+ + ContractId: string, // FOCUS 1.3+ + ContractPeriodEnd: datetime, // FOCUS 1.3+ + ContractPeriodStart: datetime, // FOCUS 1.3+ + InvoiceIssuerName: string, // FOCUS 1.3+ + PricingCurrency: string, // FOCUS 1.3+ + PricingCurrencyContractCommitmentCost: real, // FOCUS 1.4+ + ServiceProviderName: string, // FOCUS 1.4+ + x_IngestionTime: datetime, // Hubs add-on + x_SourceName: string, // Hubs add-on + x_SourceProvider: string, // Hubs add-on + x_SourceType: string, // Hubs add-on + x_SourceVersion: string // Hubs add-on ) // Update policy for ContractCommitments_raw -> ContractCommitments_final_v1_4 table