From f70c43d88579087110b353769e8b63c18d98b837 Mon Sep 17 00:00:00 2001 From: Michael Flanakin Date: Wed, 6 May 2026 08:21:44 -0700 Subject: [PATCH 1/4] Add FOCUS 1.3 hub functions (#2127) Adds HubSetup_v1_3.kql with Costs_v1_3, Prices_v1_3, CommitmentDiscountUsage_v1_3, Recommendations_v1_3, and Transactions_v1_3 that union the new Costs_final_v1_3 with the existing Costs_final_v1_2 and Costs_final_v1_0 tables. For v1_2 data unioned into the v1_3 view, the 8 new FOCUS 1.3 columns default to null/empty and ServiceProviderName/HostProviderName are populated from the deprecated ProviderName/PublisherName for back compat. The same defaults apply to the v1_0 union arm on top of the existing v1_0 -> v1_2 conversion. HubSetup_Latest.kql aliases now point to *_v1_3 functions so the unversioned Costs(), Prices(), etc. return the latest schema. Co-Authored-By: Claude Opus 4.7 (1M context) --- src/templates/finops-hub/.build.config | 1 + .../Microsoft.FinOpsHubs/Analytics/app.bicep | 1 + .../Analytics/scripts/HubSetup_Latest.kql | 10 +- .../Analytics/scripts/HubSetup_v1_3.kql | 618 ++++++++++++++++++ 4 files changed, 625 insertions(+), 5 deletions(-) create mode 100644 src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/HubSetup_v1_3.kql diff --git a/src/templates/finops-hub/.build.config b/src/templates/finops-hub/.build.config index 0963fb494..4eff22919 100644 --- a/src/templates/finops-hub/.build.config +++ b/src/templates/finops-hub/.build.config @@ -44,6 +44,7 @@ "modules/Microsoft.FinOpsHubs/Analytics/scripts/HubSetup_OpenData.kql", "modules/Microsoft.FinOpsHubs/Analytics/scripts/HubSetup_v1_0.kql", "modules/Microsoft.FinOpsHubs/Analytics/scripts/HubSetup_v1_2.kql", + "modules/Microsoft.FinOpsHubs/Analytics/scripts/HubSetup_v1_3.kql", "modules/Microsoft.FinOpsHubs/Analytics/scripts/HubSetup_Latest.kql" ] } diff --git a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/app.bicep b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/app.bicep index 42cd2738b..ebf106beb 100644 --- a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/app.bicep +++ b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/app.bicep @@ -414,6 +414,7 @@ module hub_VersionedScripts '../../fx/hub-database.bicep' = if (useAzure) { scripts: { v1_0: loadTextContent('scripts/HubSetup_v1_0.kql') v1_2: loadTextContent('scripts/HubSetup_v1_2.kql') + v1_3: loadTextContent('scripts/HubSetup_v1_3.kql') } continueOnErrors: continueOnErrors forceUpdateTag: forceUpdateTag diff --git a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/HubSetup_Latest.kql b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/HubSetup_Latest.kql index 8de86a1aa..1895b39db 100644 --- a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/HubSetup_Latest.kql +++ b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/HubSetup_Latest.kql @@ -13,7 +13,7 @@ with (docstring = 'Gets all commitment discount usage records with the latest supported version of the FOCUS schema.', folder = 'CommitmentDiscountUsage') CommitmentDiscountUsage() { - CommitmentDiscountUsage_v1_2() + CommitmentDiscountUsage_v1_3() } @@ -21,7 +21,7 @@ CommitmentDiscountUsage() with (docstring = 'Gets all cost and usage records with the latest supported version of the FOCUS schema.', folder = 'Costs') Costs() { - Costs_v1_2() + Costs_v1_3() } @@ -29,7 +29,7 @@ Costs() with (docstring = 'Gets all prices with the latest supported version of the FOCUS schema.', folder = 'Prices') Prices() { - Prices_v1_2() + Prices_v1_3() } @@ -37,7 +37,7 @@ Prices() with (docstring = 'Gets all recommendations with the latest supported version of the FOCUS schema.', folder = 'Recommendations') Recommendations() { - Recommendations_v1_2() + Recommendations_v1_3() } @@ -45,5 +45,5 @@ Recommendations() with (docstring = 'Gets all transactions with the latest supported version of the FOCUS schema.', folder = 'Transactions') Transactions() { - Transactions_v1_2() + Transactions_v1_3() } diff --git a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/HubSetup_v1_3.kql b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/HubSetup_v1_3.kql new file mode 100644 index 000000000..62995ffa5 --- /dev/null +++ b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/HubSetup_v1_3.kql @@ -0,0 +1,618 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//====================================================================================================================== +// Hub database / FOCUS 1.3 functions +// Used for reporting with backward compatibility. +//====================================================================================================================== + +// For allowed commands, see https://learn.microsoft.com/azure/data-explorer/database-script + + +// CommitmentDiscountUsage_final_v1_3 +.create-or-alter function +with (docstring = 'Gets all commitment discount usage records aligned to FOCUS 1.3.', folder = 'CommitmentDiscountUsage') +CommitmentDiscountUsage_v1_3() +{ + database('Ingestion').CommitmentDiscountUsage_final_v1_3 + | union ( + database('Ingestion').CommitmentDiscountUsage_final_v1_0 + // Convert decimal to real + | extend + ConsumedQuantity = toreal(ConsumedQuantity), + x_CommitmentDiscountCommittedCount = toreal(x_CommitmentDiscountCommittedCount), + x_CommitmentDiscountCommittedAmount = toreal(x_CommitmentDiscountCommittedAmount), + x_CommitmentDiscountNormalizedRatio = toreal(x_CommitmentDiscountNormalizedRatio) + // Add new columns + | lookup kind=leftouter (Services | distinct x_ResourceType, ServiceSubcategory) on x_ResourceType + | extend CommitmentDiscountQuantity = ConsumedQuantity * x_CommitmentDiscountNormalizedRatio + | extend CommitmentDiscountUnit = case( + x_CommitmentDiscountNormalizedRatio == 1, 'Hours', + x_CommitmentDiscountNormalizedRatio > 1, 'Normalized Hours', + '' + ) + ) + | project + ChargePeriodEnd, + ChargePeriodStart, + CommitmentDiscountCategory, + CommitmentDiscountId, + CommitmentDiscountQuantity, + CommitmentDiscountType, + CommitmentDiscountUnit, + ConsumedQuantity, + ProviderName, + ResourceId, + ResourceName, + ResourceType, + ServiceCategory, + ServiceName, + ServiceSubcategory, + SubAccountId, + x_CommitmentDiscountCommittedCount, + x_CommitmentDiscountCommittedAmount, + x_CommitmentDiscountNormalizedGroup, + x_CommitmentDiscountNormalizedRatio, + x_IngestionTime, + x_ResourceGroupName, + x_ResourceType, + x_ServiceModel, + x_SkuOrderId, + x_SkuSize, + x_SourceName, + x_SourceProvider, + x_SourceType, + x_SourceVersion +} + + +// Costs_final_v1_3 +.create-or-alter function +with (docstring = 'Gets all cost and usage records aligned to FOCUS 1.3.', folder = 'Costs') +Costs_v1_3() +{ + database('Ingestion').Costs_final_v1_3 + | union ( + database('Ingestion').Costs_final_v1_0 + // Convert decimal to real + | extend + BilledCost = toreal(BilledCost), + ConsumedQuantity = toreal(ConsumedQuantity), + ContractedCost = toreal(ContractedCost), + ContractedUnitPrice = toreal(ContractedUnitPrice), + EffectiveCost = toreal(EffectiveCost), + ListCost = toreal(ListCost), + ListUnitPrice = toreal(ListUnitPrice), + PricingQuantity = toreal(PricingQuantity), + x_BilledCostInUsd = toreal(x_BilledCostInUsd), + x_BilledUnitPrice = toreal(x_BilledUnitPrice), + x_BillingExchangeRate = toreal(x_BillingExchangeRate), + x_ContractedCostInUsd = toreal(x_ContractedCostInUsd), + x_CurrencyConversionRate = toreal(x_CurrencyConversionRate), + x_EffectiveCostInUsd = toreal(x_EffectiveCostInUsd), + x_EffectiveUnitPrice = toreal(x_EffectiveUnitPrice), + x_ListCostInUsd = toreal(x_ListCostInUsd), + x_PricingBlockSize = toreal(x_PricingBlockSize) + // Rename columns + | project-rename + InvoiceId = x_InvoiceId, + PricingCurrency = x_PricingCurrency, + SkuMeter = x_SkuMeterName + // Add new columns + | lookup kind=leftouter (Services | where isnotempty(x_ResourceType) | summarize take_any(ServiceSubcategory), take_any(x_ServiceModel) by x_ResourceType) on x_ResourceType + | extend CapacityReservationId = tostring(x_SkuDetails.VMCapacityReservationId) + | extend CapacityReservationStatus = case( + isempty(CapacityReservationId), '', + tolower(x_ResourceType) == 'microsoft.compute/capacityreservationgroups/capacityreservations', 'Unused', + 'Used' + ) + | extend x_CommitmentDiscountNormalizedRatio = case( + // Not applicable + isempty(CommitmentDiscountStatus), real(null), + // Parse from SKU details if not specified explicitly + toreal(coalesce(x_SkuDetails.RINormalizationRatio, dynamic(1))) + ) + | extend CommitmentDiscountQuantity = case( + isempty(CommitmentDiscountStatus), real(null), + CommitmentDiscountCategory == 'Spend', EffectiveCost / coalesce(x_BillingExchangeRate, real(1)), + CommitmentDiscountCategory == 'Usage' and isnotempty(x_CommitmentDiscountNormalizedRatio), PricingQuantity / coalesce(x_PricingBlockSize, real(1)) * x_CommitmentDiscountNormalizedRatio, + real(null) + ) + | extend CommitmentDiscountUnit = case( + isempty(CommitmentDiscountQuantity), '', + CommitmentDiscountCategory == 'Spend', PricingCurrency, + CommitmentDiscountCategory == 'Usage' and x_CommitmentDiscountNormalizedRatio == real(1), ConsumedUnit, + CommitmentDiscountCategory == 'Usage' and x_CommitmentDiscountNormalizedRatio > real(1), strcat('Normalized ', ConsumedUnit), + '' + ) + | extend x_AmortizationClass = case( + ChargeCategory == 'Purchase' and (tolower(ResourceId) contains '/microsoft.capacity/reservationorders/' or tolower(ResourceId) contains '/microsoft.billingbenefits/savingsplanorders/'), 'Principal', + ChargeCategory == 'Usage' and isnotempty(CommitmentDiscountId) and isnotempty(CommitmentDiscountStatus), 'Amortized Charge', + '' + ) + // Hubs add-ons + | extend x_CommitmentDiscountUtilizationPotential = case( + ChargeCategory == 'Purchase', real(0), + ProviderName == 'Microsoft' and isnotempty(CommitmentDiscountCategory), EffectiveCost, + CommitmentDiscountCategory == 'Usage', ConsumedQuantity, + CommitmentDiscountCategory == 'Spend', EffectiveCost, + real(0) + ) + | extend x_CommitmentDiscountUtilizationAmount = iff(CommitmentDiscountStatus == 'Used', x_CommitmentDiscountUtilizationPotential, real(0)) + | extend x_SkuCoreCount = toint(coalesce(x_SkuDetails.VCPUs, x_SkuDetails.VCores, x_SkuDetails.vCores)) + | extend x_SkuInstanceType = tostring(coalesce(x_SkuDetails.ServiceType, x_SkuDetails.ServerSku)) + | extend x_SkuOperatingSystem = case( + x_SkuDetails.ImageType == 'Canonical', 'Linux', + x_SkuDetails.ImageType == 'Windows Server BYOL', 'Windows Server', + x_SkuMeterSubcategory endswith ' Series Windows', 'Windows Server', + x_SkuDetails.ImageType + ) + | extend x_ConsumedCoreHours = iff(ConsumedUnit == 'Hours' and isnotempty(x_SkuCoreCount), x_SkuCoreCount * ConsumedQuantity, real(null)) + | extend tmp_SqlAhb = tolower(x_SkuDetails.AHB) + | extend x_SkuLicenseType = case( + x_SkuDetails.ImageType contains 'Windows Server BYOL', 'Windows Server', + x_SkuMeterSubcategory == 'SQL Server Azure Hybrid Benefit', 'SQL Server', + '' + ) + | extend x_SkuLicenseStatus = case( + isnotempty(x_SkuLicenseType) or tmp_SqlAhb == 'true' or (x_SkuMeterSubcategory contains 'Azure Hybrid Benefit'), 'Enabled', + (x_SkuMeterSubcategory contains 'Windows') or tmp_SqlAhb == 'false', 'Not enabled', + '' + ) + | extend x_SkuLicenseQuantity = case( + isempty(x_SkuCoreCount), int(null), + x_SkuCoreCount <= 8, int(8), + x_SkuCoreCount > 8, x_SkuCoreCount, + int(null) + ) + | extend x_SkuLicenseUnit = iff(isnotempty(x_SkuLicenseQuantity), 'Cores', '') + | extend x_CommitmentDiscountSavings = iff(ContractedCost < EffectiveCost, real(0), ContractedCost - EffectiveCost) + | extend x_NegotiatedDiscountSavings = iff(ListCost < ContractedCost, real(0), ListCost - ContractedCost) + | extend x_TotalSavings = iff(ListCost < EffectiveCost, real(0), ListCost - EffectiveCost) + | extend x_CommitmentDiscountPercent = iff(ContractedUnitPrice == 0, real(0), (ContractedUnitPrice - x_EffectiveUnitPrice) / ContractedUnitPrice) + | extend x_NegotiatedDiscountPercent = iff(ListUnitPrice == 0, real(0), (ListUnitPrice - ContractedUnitPrice) / ListUnitPrice) + | extend x_TotalDiscountPercent = iff(ListUnitPrice == 0, real(0), (ListUnitPrice - x_EffectiveUnitPrice) / ListUnitPrice) + // SkuPriceDetails conversion -- Must be after hubs add-ons + | extend SkuPriceDetails = parse_json(replace_regex(replace_regex(replace_regex(replace_regex(replace_regex(replace_regex(tostring(x_SkuDetails) + // Prefix all keys with x_ first to avoid double-prefixing + , @'([\{,])"', @'\1"x_') + // CoreCount for number of CPUs/vCPUs/cores/vCores + , @'"x_(VCPUs|VCores|vCores)":', @'"CoreCount":') + // TODO: DiskMaxIops for disk I/O operations per second (IOPS) + // TODO: DiskSpace for disk size in GiB + // TODO: DiskType for the kind of disk (e.g., SSD, HDD, NVMe) + // TODO: GpuCount for the number of GPUs + // InstanceType for the resource size/SKU (e.g., ArmSkuName) + , @'"x_(ServerSku|ServiceType)":', @'"InstanceType":') + // TODO: InstanceSeries for the size family/series + // TODO: MemorySize for the RAM in GiB + // TODO: NetworkMaxIops for network I/O operations per second (IOPS) + // TODO: NetworkMaxThroughput for network max throughput for data transfer in Mbps + // OperatingSystem for the OS name + , @'("x_ImageType":"Canonical")', @'\1,"OperatingSystem":"Linux"') + , @'("x_ImageType":"Windows Server( BYOL)?")', @'\1,"OperatingSystem":"Windows Server"') + , @'("x_ImageType":("[^"]+"))', @'\1,"OperatingSystem":\2') + // TODO: Redundancy for the level of redundancy (e.g., Local, Zonal, Global) + // TODO: StorageClass for the tier of storage (e.g., Hot, Archive, Nearline) + ) + | extend SkuPriceDetails = iff(isempty(SkuPriceDetails.OperatingSystem) and isnotempty(x_SkuOperatingSystem), + parse_json(replace_string(tostring(SkuPriceDetails), '}', strcat(@',"OperatingSystem":"', x_SkuOperatingSystem, '"}'))), + SkuPriceDetails) + // FOCUS 1.3 column defaults (data was ingested under v1_0 schema) + | extend AllocatedMethodId = '' + | extend AllocatedMethodDetails = dynamic(null) + | extend AllocatedResourceId = '' + | extend AllocatedResourceName = '' + | extend AllocatedTags = dynamic(null) + | extend ContractApplied = dynamic(null) + | extend ServiceProviderName = ProviderName + | extend HostProviderName = PublisherName + ) + | union ( + database('Ingestion').Costs_final_v1_2 + // FOCUS 1.3 column defaults (data was ingested under v1_2 schema) + | extend AllocatedMethodId = '' + | extend AllocatedMethodDetails = dynamic(null) + | extend AllocatedResourceId = '' + | extend AllocatedResourceName = '' + | extend AllocatedTags = dynamic(null) + | extend ContractApplied = dynamic(null) + | extend ServiceProviderName = ProviderName + | extend HostProviderName = PublisherName + ) + | extend SkuPriceDetails = iff(isnotempty(SkuPriceDetails), SkuPriceDetails, parse_json(replace_regex(tostring(x_SkuDetails), @'([\{,])"', @'\1"x_'))) + | project + AllocatedMethodDetails, + AllocatedMethodId, + AllocatedResourceId, + AllocatedResourceName, + AllocatedTags, + AvailabilityZone, + BilledCost, + BillingAccountId, + BillingAccountName, + BillingAccountType, + BillingCurrency, + BillingPeriodEnd, + BillingPeriodStart, + CapacityReservationId, + CapacityReservationStatus, + ChargeCategory, + ChargeClass, + ChargeDescription, + ChargeFrequency, + ChargePeriodEnd, + ChargePeriodStart, + CommitmentDiscountCategory, + CommitmentDiscountId, + CommitmentDiscountName, + CommitmentDiscountQuantity, + CommitmentDiscountStatus, + CommitmentDiscountType, + CommitmentDiscountUnit, + ConsumedQuantity, + ConsumedUnit, + ContractApplied, + ContractedCost, + ContractedUnitPrice, + EffectiveCost, + HostProviderName, + InvoiceId, + InvoiceIssuerName, + ListCost, + ListUnitPrice, + PricingCategory, + PricingCurrency, + PricingQuantity, + PricingUnit, + ProviderName, + PublisherName, + RegionId, + RegionName, + ResourceId, + ResourceName, + ResourceType, + ServiceCategory, + ServiceName, + ServiceProviderName, + ServiceSubcategory, + SkuId, + SkuMeter, + SkuPriceDetails, + SkuPriceId, + SubAccountId, + SubAccountName, + SubAccountType, + Tags, + x_AccountId, + x_AccountName, + x_AccountOwnerId, + x_AmortizationClass, + x_BilledCostInUsd, + x_BilledUnitPrice, + x_BillingAccountAgreement, + x_BillingAccountId, + x_BillingAccountName, + x_BillingExchangeRate, + x_BillingExchangeRateDate, + x_BillingItemCode, + x_BillingItemName, + x_BillingProfileId, + x_BillingProfileName, + x_ChargeId, + x_CommitmentDiscountNormalizedRatio, + x_CommitmentDiscountPercent, + x_CommitmentDiscountSavings, + x_CommitmentDiscountSpendEligibility, + x_CommitmentDiscountUsageEligibility, + x_CommitmentDiscountUtilizationAmount, + x_CommitmentDiscountUtilizationPotential, + x_CommodityCode, + x_CommodityName, + x_ComponentName, + x_ComponentType, + x_ConsumedCoreHours, + x_ContractedCostInUsd, + x_CostAllocationRuleName, + x_CostCategories, + x_CostCenter, + x_CostType, + x_Credits, + x_CurrencyConversionRate, + x_CustomerId, + x_CustomerName, + x_Discount, + x_EffectiveCostInUsd, + x_EffectiveUnitPrice, + x_ExportTime, + x_IngestionTime, + x_InstanceID, + x_InvoiceIssuerId, + x_InvoiceSectionId, + x_InvoiceSectionName, + x_ListCostInUsd, + x_Location, + x_NegotiatedDiscountPercent, + x_NegotiatedDiscountSavings, + x_Operation, + x_OwnerAccountID, + x_PartnerCreditApplied, + x_PartnerCreditRate, + x_PricingBlockSize, + x_PricingSubcategory, + x_PricingUnitDescription, + x_Project, + x_PublisherCategory, + x_PublisherId, + x_ResellerId, + x_ResellerName, + x_ResourceGroupName, + x_ResourceType, + x_ServiceCode, + x_ServiceId, + x_ServiceModel, + x_ServicePeriodEnd, + x_ServicePeriodStart, + x_SkuCoreCount, + x_SkuDescription, + x_SkuDetails, + x_SkuInstanceType, + x_SkuIsCreditEligible, + x_SkuLicenseQuantity, + x_SkuLicenseStatus, + x_SkuLicenseType, + x_SkuLicenseUnit, + x_SkuMeterCategory, + x_SkuMeterId, + x_SkuMeterSubcategory, + x_SkuOfferId, + x_SkuOperatingSystem, + x_SkuOrderId, + x_SkuOrderName, + x_SkuPartNumber, + x_SkuPlanName, + x_SkuRegion, + x_SkuServiceFamily, + x_SkuTerm, + x_SkuTier, + x_SourceChanges, + x_SourceName, + x_SourceProvider, + x_SourceType, + x_SourceValues, + x_SourceVersion, + x_SubproductName, + x_TotalDiscountPercent, + x_TotalSavings, + x_UsageType +} + + +// Prices_final_v1_3 +.create-or-alter function +with (docstring = 'Gets all prices aligned to FOCUS 1.3.', folder = 'Prices') +Prices_v1_3() +{ + database('Ingestion').Prices_final_v1_3 + | union ( + database('Ingestion').Prices_final_v1_0 + // Convert decimal to real + | extend + ContractedUnitPrice = toreal(ContractedUnitPrice), + ListUnitPrice = toreal(ListUnitPrice), + x_BaseUnitPrice = toreal(x_BaseUnitPrice), + x_ContractedUnitPriceDiscount = toreal(x_ContractedUnitPriceDiscount), + x_ContractedUnitPriceDiscountPercent = toreal(x_ContractedUnitPriceDiscountPercent), + x_EffectiveUnitPrice = toreal(x_EffectiveUnitPrice), + x_EffectiveUnitPriceDiscount = toreal(x_EffectiveUnitPriceDiscount), + x_EffectiveUnitPriceDiscountPercent = toreal(x_EffectiveUnitPriceDiscountPercent), + x_PricingBlockSize = toreal(x_PricingBlockSize), + x_SkuIncludedQuantity = toreal(x_SkuIncludedQuantity), + x_SkuTier = toreal(x_SkuTier), + x_TotalUnitPriceDiscount = toreal(x_TotalUnitPriceDiscount), + x_TotalUnitPriceDiscountPercent = toreal(x_TotalUnitPriceDiscountPercent) + // Rename columns + | project-rename + PricingCurrency = x_PricingCurrency, + SkuMeter = x_SkuMeterName + ) + | project + BillingAccountId, + BillingAccountName, + BillingCurrency, + ChargeCategory, + CommitmentDiscountCategory, + CommitmentDiscountType, + CommitmentDiscountUnit, + ContractedUnitPrice, + ListUnitPrice, + PricingCategory, + PricingCurrency, + PricingUnit, + SkuId, + SkuMeter, + SkuPriceId, + SkuPriceIdv2, + x_BaseUnitPrice, + x_BillingAccountAgreement, + x_BillingAccountId, + x_BillingProfileId, + x_CommitmentDiscountNormalizedRatio, + x_CommitmentDiscountSpendEligibility, + x_CommitmentDiscountUsageEligibility, + x_ContractedUnitPriceDiscount, + x_ContractedUnitPriceDiscountPercent, + x_EffectivePeriodEnd, + x_EffectivePeriodStart, + x_EffectiveUnitPrice, + x_EffectiveUnitPriceDiscount, + x_EffectiveUnitPriceDiscountPercent, + x_IngestionTime, + x_PricingBlockSize, + x_PricingSubcategory, + x_PricingUnitDescription, + x_SkuDescription, + x_SkuId, + x_SkuIncludedQuantity, + x_SkuMeterCategory, + x_SkuMeterId, + x_SkuMeterSubcategory, + x_SkuMeterType, + x_SkuPriceType, + x_SkuProductId, + x_SkuRegion, + x_SkuServiceFamily, + x_SkuOfferId, + x_SkuPartNumber, + x_SkuTerm, + x_SkuTier, + x_SourceName, + x_SourceProvider, + x_SourceType, + x_SourceVersion, + x_TotalUnitPriceDiscount, + x_TotalUnitPriceDiscountPercent +} + + +// Recommendations_final_v1_3 +.create-or-alter function +with (docstring = 'Gets all recommendations aligned to FOCUS 1.3.', folder = 'Recommendations') +Recommendations_v1_3() +{ + database('Ingestion').Recommendations_final_v1_3 + | union ( + database('Ingestion').Recommendations_final_v1_0 + // Convert decimal to real + | extend + x_EffectiveCostAfter = toreal(x_EffectiveCostAfter), + x_EffectiveCostBefore = toreal(x_EffectiveCostBefore), + x_EffectiveCostSavings = toreal(x_EffectiveCostSavings) + ) + | project + ProviderName, + ResourceId, + ResourceName, + ResourceType, + SubAccountId, + SubAccountName, + x_EffectiveCostAfter, + x_EffectiveCostBefore, + x_EffectiveCostSavings, + x_IngestionTime, + x_RecommendationCategory, + x_RecommendationDate, + x_RecommendationDescription, + x_RecommendationDetails, + x_RecommendationId, + x_ResourceGroupName, + x_SourceName, + x_SourceProvider, + x_SourceType, + x_SourceVersion +} + + +// Transactions_final_v1_3 +.create-or-alter function +with (docstring = 'Gets all transactions aligned to FOCUS 1.3.', folder = 'Transactions') +Transactions_v1_3() +{ + database('Ingestion').Transactions_final_v1_3 + | union ( + database('Ingestion').Transactions_final_v1_0 + // Convert decimal to real + | extend + BilledCost = toreal(BilledCost), + PricingQuantity = toreal(PricingQuantity), + x_MonetaryCommitment = toreal(x_MonetaryCommitment), + x_Overage = toreal(x_Overage) + // Rename columns + | project-rename + InvoiceId = x_InvoiceId + ) + | project + BilledCost, + BillingAccountId, + BillingAccountName, + BillingCurrency, + BillingPeriodEnd, + BillingPeriodStart, + ChargeCategory, + ChargeClass, + ChargeDescription, + ChargeFrequency, + ChargePeriodStart, + InvoiceId, + PricingQuantity, + PricingUnit, + ProviderName, + RegionId, + RegionName, + SubAccountId, + SubAccountName, + x_AccountName, + x_AccountOwnerId, + x_CostCenter, + x_InvoiceNumber, + x_InvoiceSectionId, + x_InvoiceSectionName, + x_IngestionTime, + x_MonetaryCommitment, + x_Overage, + x_PurchasingBillingAccountId, + x_SkuOrderId, + x_SkuOrderName, + x_SkuSize, + x_SkuTerm, + x_SourceName, + x_SourceProvider, + x_SourceType, + x_SourceVersion, + x_SubscriptionId, + x_TransactionType +} + + +//====================================================================================================================== +// Latest FOCUS version +//====================================================================================================================== + +.create-or-alter function +with (docstring = 'Gets all commitment discount usage records with the latest supported version of the FOCUS schema.', folder = 'CommitmentDiscountUsage') +CommitmentDiscountUsage() +{ + CommitmentDiscountUsage_v1_3() +} + + +.create-or-alter function +with (docstring = 'Gets all cost and usage records with the latest supported version of the FOCUS schema.', folder = 'Costs') +Costs() +{ + Costs_v1_3() +} + + +.create-or-alter function +with (docstring = 'Gets all prices with the latest supported version of the FOCUS schema.', folder = 'Prices') +Prices() +{ + Prices_v1_3() +} + + +.create-or-alter function +with (docstring = 'Gets all recommendations with the latest supported version of the FOCUS schema.', folder = 'Recommendations') +Recommendations() +{ + Recommendations_v1_3() +} + + +.create-or-alter function +with (docstring = 'Gets all transactions with the latest supported version of the FOCUS schema.', folder = 'Transactions') +Transactions() +{ + Transactions_v1_3() +} From 8a01133a132a6d130057a04c95515c3c4cc352f3 Mon Sep 17 00:00:00 2001 From: Michael Flanakin Date: Sun, 17 May 2026 13:25:49 -0700 Subject: [PATCH 2/4] Add FOCUS 1.4 hub functions (HubSetup_v1_4.kql) Renames HubSetup_v1_3.kql to HubSetup_v1_4.kql, consolidating all hub functions (including ContractCommitment) for FOCUS 1.4 GA. Updates HubSetup_Latest.kql to alias all unversioned functions to v1_4. Updates app.bicep and .build.config to reference v1_4. Co-Authored-By: Claude --- src/templates/finops-hub/.build.config | 2 +- .../Microsoft.FinOpsHubs/Analytics/app.bicep | 2 +- .../Analytics/scripts/HubSetup_Latest.kql | 18 ++-- .../{HubSetup_v1_3.kql => HubSetup_v1_4.kql} | 88 +++++++++++++------ 4 files changed, 75 insertions(+), 35 deletions(-) rename src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/{HubSetup_v1_3.kql => HubSetup_v1_4.kql} (89%) diff --git a/src/templates/finops-hub/.build.config b/src/templates/finops-hub/.build.config index c8091cb04..e4807f5f0 100644 --- a/src/templates/finops-hub/.build.config +++ b/src/templates/finops-hub/.build.config @@ -44,7 +44,7 @@ "modules/Microsoft.FinOpsHubs/Analytics/scripts/HubSetup_OpenData.kql", "modules/Microsoft.FinOpsHubs/Analytics/scripts/HubSetup_v1_0.kql", "modules/Microsoft.FinOpsHubs/Analytics/scripts/HubSetup_v1_2.kql", - "modules/Microsoft.FinOpsHubs/Analytics/scripts/HubSetup_v1_3.kql", + "modules/Microsoft.FinOpsHubs/Analytics/scripts/HubSetup_v1_4.kql", "modules/Microsoft.FinOpsHubs/Analytics/scripts/HubSetup_Latest.kql" ] } diff --git a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/app.bicep b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/app.bicep index 4cede47fe..f374c677c 100644 --- a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/app.bicep +++ b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/app.bicep @@ -414,7 +414,7 @@ module hub_VersionedScripts '../../fx/hub-database.bicep' = if (useAzure) { scripts: { v1_0: loadTextContent('scripts/HubSetup_v1_0.kql') v1_2: loadTextContent('scripts/HubSetup_v1_2.kql') - v1_3: loadTextContent('scripts/HubSetup_v1_3.kql') + v1_4: loadTextContent('scripts/HubSetup_v1_4.kql') } continueOnErrors: continueOnErrors forceUpdateTag: forceUpdateTag diff --git a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/HubSetup_Latest.kql b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/HubSetup_Latest.kql index 1895b39db..c3a4985f2 100644 --- a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/HubSetup_Latest.kql +++ b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/HubSetup_Latest.kql @@ -13,7 +13,15 @@ with (docstring = 'Gets all commitment discount usage records with the latest supported version of the FOCUS schema.', folder = 'CommitmentDiscountUsage') CommitmentDiscountUsage() { - CommitmentDiscountUsage_v1_3() + CommitmentDiscountUsage_v1_4() +} + + +.create-or-alter function +with (docstring = 'Gets all contract commitments with the latest supported version of the FOCUS schema.', folder = 'ContractCommitment') +ContractCommitment() +{ + ContractCommitment_v1_4() } @@ -21,7 +29,7 @@ CommitmentDiscountUsage() with (docstring = 'Gets all cost and usage records with the latest supported version of the FOCUS schema.', folder = 'Costs') Costs() { - Costs_v1_3() + Costs_v1_4() } @@ -29,7 +37,7 @@ Costs() with (docstring = 'Gets all prices with the latest supported version of the FOCUS schema.', folder = 'Prices') Prices() { - Prices_v1_3() + Prices_v1_4() } @@ -37,7 +45,7 @@ Prices() with (docstring = 'Gets all recommendations with the latest supported version of the FOCUS schema.', folder = 'Recommendations') Recommendations() { - Recommendations_v1_3() + Recommendations_v1_4() } @@ -45,5 +53,5 @@ Recommendations() with (docstring = 'Gets all transactions with the latest supported version of the FOCUS schema.', folder = 'Transactions') Transactions() { - Transactions_v1_3() + Transactions_v1_4() } diff --git a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/HubSetup_v1_3.kql b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/HubSetup_v1_4.kql similarity index 89% rename from src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/HubSetup_v1_3.kql rename to src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/HubSetup_v1_4.kql index 62995ffa5..cabdbdfc5 100644 --- a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/HubSetup_v1_3.kql +++ b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/HubSetup_v1_4.kql @@ -2,19 +2,19 @@ // Licensed under the MIT License. //====================================================================================================================== -// Hub database / FOCUS 1.3 functions +// Hub database / FOCUS 1.4 functions // Used for reporting with backward compatibility. //====================================================================================================================== // For allowed commands, see https://learn.microsoft.com/azure/data-explorer/database-script -// CommitmentDiscountUsage_final_v1_3 +// CommitmentDiscountUsage_final_v1_4 .create-or-alter function -with (docstring = 'Gets all commitment discount usage records aligned to FOCUS 1.3.', folder = 'CommitmentDiscountUsage') -CommitmentDiscountUsage_v1_3() +with (docstring = 'Gets all commitment discount usage records aligned to FOCUS 1.4.', folder = 'CommitmentDiscountUsage') +CommitmentDiscountUsage_v1_4() { - database('Ingestion').CommitmentDiscountUsage_final_v1_3 + database('Ingestion').CommitmentDiscountUsage_final_v1_4 | union ( database('Ingestion').CommitmentDiscountUsage_final_v1_0 // Convert decimal to real @@ -66,12 +66,39 @@ CommitmentDiscountUsage_v1_3() } -// Costs_final_v1_3 +// ContractCommitment_final_v1_4 .create-or-alter function -with (docstring = 'Gets all cost and usage records aligned to FOCUS 1.3.', folder = 'Costs') -Costs_v1_3() +with (docstring = 'Gets all contract commitments aligned to FOCUS 1.4.', folder = 'ContractCommitment') +ContractCommitment_v1_4() { - database('Ingestion').Costs_final_v1_3 + database('Ingestion').ContractCommitment_final_v1_4 + | union ( + database('Ingestion').ContractCommitment_final_v1_3 + // FOCUS 1.4 column defaults (data was ingested under v1_3 schema) + | extend BenefitCategory = '' + | extend ContractCommitmentApplicability = dynamic(null) + | extend Created = datetime(null) + | extend DiscountPercentage = real(null) + | extend DurationType = '' + | extend FulfillmentInterval = '' + | extend LastUpdated = datetime(null) + | extend LifecycleStatus = '' + | extend Model = '' + | extend OfferCategory = '' + | extend PaymentInterval = '' + | extend PaymentModel = '' + | extend PaymentUpfrontPercentage = real(null) + | extend PricingCurrencyContractCommitmentCost = real(null) + ) +} + + +// Costs_final_v1_4 +.create-or-alter function +with (docstring = 'Gets all cost and usage records aligned to FOCUS 1.4.', folder = 'Costs') +Costs_v1_4() +{ + database('Ingestion').Costs_final_v1_4 | union ( database('Ingestion').Costs_final_v1_0 // Convert decimal to real @@ -220,6 +247,12 @@ Costs_v1_3() | extend ServiceProviderName = ProviderName | extend HostProviderName = PublisherName ) + | union ( + database('Ingestion').Costs_final_v1_3 + // No column adds needed; v1_3 already has the FOCUS 1.3 columns. The + // deprecated ProviderName / PublisherName fall off via the | project + // list below since FOCUS 1.4 removes them. + ) | extend SkuPriceDetails = iff(isnotempty(SkuPriceDetails), SkuPriceDetails, parse_json(replace_regex(tostring(x_SkuDetails), @'([\{,])"', @'\1"x_'))) | project AllocatedMethodDetails, @@ -265,8 +298,7 @@ Costs_v1_3() PricingCurrency, PricingQuantity, PricingUnit, - ProviderName, - PublisherName, + // ProviderName and PublisherName removed in FOCUS 1.4. Use ServiceProviderName / HostProviderName. RegionId, RegionName, ResourceId, @@ -388,12 +420,12 @@ Costs_v1_3() } -// Prices_final_v1_3 +// Prices_final_v1_4 .create-or-alter function -with (docstring = 'Gets all prices aligned to FOCUS 1.3.', folder = 'Prices') -Prices_v1_3() +with (docstring = 'Gets all prices aligned to FOCUS 1.4.', folder = 'Prices') +Prices_v1_4() { - database('Ingestion').Prices_final_v1_3 + database('Ingestion').Prices_final_v1_4 | union ( database('Ingestion').Prices_final_v1_0 // Convert decimal to real @@ -475,12 +507,12 @@ Prices_v1_3() } -// Recommendations_final_v1_3 +// Recommendations_final_v1_4 .create-or-alter function -with (docstring = 'Gets all recommendations aligned to FOCUS 1.3.', folder = 'Recommendations') -Recommendations_v1_3() +with (docstring = 'Gets all recommendations aligned to FOCUS 1.4.', folder = 'Recommendations') +Recommendations_v1_4() { - database('Ingestion').Recommendations_final_v1_3 + database('Ingestion').Recommendations_final_v1_4 | union ( database('Ingestion').Recommendations_final_v1_0 // Convert decimal to real @@ -513,12 +545,12 @@ Recommendations_v1_3() } -// Transactions_final_v1_3 +// Transactions_final_v1_4 .create-or-alter function -with (docstring = 'Gets all transactions aligned to FOCUS 1.3.', folder = 'Transactions') -Transactions_v1_3() +with (docstring = 'Gets all transactions aligned to FOCUS 1.4.', folder = 'Transactions') +Transactions_v1_4() { - database('Ingestion').Transactions_final_v1_3 + database('Ingestion').Transactions_final_v1_4 | union ( database('Ingestion').Transactions_final_v1_0 // Convert decimal to real @@ -582,7 +614,7 @@ Transactions_v1_3() with (docstring = 'Gets all commitment discount usage records with the latest supported version of the FOCUS schema.', folder = 'CommitmentDiscountUsage') CommitmentDiscountUsage() { - CommitmentDiscountUsage_v1_3() + CommitmentDiscountUsage_v1_4() } @@ -590,7 +622,7 @@ CommitmentDiscountUsage() with (docstring = 'Gets all cost and usage records with the latest supported version of the FOCUS schema.', folder = 'Costs') Costs() { - Costs_v1_3() + Costs_v1_4() } @@ -598,7 +630,7 @@ Costs() with (docstring = 'Gets all prices with the latest supported version of the FOCUS schema.', folder = 'Prices') Prices() { - Prices_v1_3() + Prices_v1_4() } @@ -606,7 +638,7 @@ Prices() with (docstring = 'Gets all recommendations with the latest supported version of the FOCUS schema.', folder = 'Recommendations') Recommendations() { - Recommendations_v1_3() + Recommendations_v1_4() } @@ -614,5 +646,5 @@ Recommendations() with (docstring = 'Gets all transactions with the latest supported version of the FOCUS schema.', folder = 'Transactions') Transactions() { - Transactions_v1_3() + Transactions_v1_4() } From 56f9b3015a1a7df981d12a773dd2ad888a26d960 Mon Sep 17 00:00:00 2001 From: Michael Flanakin Date: Tue, 26 May 2026 00:25:01 -0700 Subject: [PATCH 3/4] Cascade PR #2126 feedback to hub functions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Rename ContractCommitment_v1_4() -> ContractCommitments_v1_4() (and update folder label) - Add BillingPeriods_v1_4() and InvoiceDetails_v1_4() hub functions - Add BillingPeriods(), ContractCommitments() (renamed), and InvoiceDetails() unversioned aliases in HubSetup_Latest.kql - Add CommitmentProgramEligibilityDetails + 12 ContractCommitment* + InvoiceDetailId column defaults to Costs_final_v1_0 and Costs_final_v1_2 union arms (back-compat shims) and add them to the project list 🤖 Generated with [Claude Code](https://claude.ai/claude-code) Co-Authored-By: Michael Flanakin Co-Authored-By: Claude --- .../Analytics/scripts/HubSetup_Latest.kql | 22 ++- .../Analytics/scripts/HubSetup_v1_4.kql | 128 +++++++++++------- 2 files changed, 101 insertions(+), 49 deletions(-) diff --git a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/HubSetup_Latest.kql b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/HubSetup_Latest.kql index c3a4985f2..bb0bc69fc 100644 --- a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/HubSetup_Latest.kql +++ b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/HubSetup_Latest.kql @@ -9,6 +9,14 @@ // For allowed commands, see https://learn.microsoft.com/azure/data-explorer/database-script +.create-or-alter function +with (docstring = 'Gets all billing periods with the latest supported version of the FOCUS schema.', folder = 'BillingPeriods') +BillingPeriods() +{ + BillingPeriods_v1_4() +} + + .create-or-alter function with (docstring = 'Gets all commitment discount usage records with the latest supported version of the FOCUS schema.', folder = 'CommitmentDiscountUsage') CommitmentDiscountUsage() @@ -18,10 +26,10 @@ CommitmentDiscountUsage() .create-or-alter function -with (docstring = 'Gets all contract commitments with the latest supported version of the FOCUS schema.', folder = 'ContractCommitment') -ContractCommitment() +with (docstring = 'Gets all contract commitments with the latest supported version of the FOCUS schema.', folder = 'ContractCommitments') +ContractCommitments() { - ContractCommitment_v1_4() + ContractCommitments_v1_4() } @@ -33,6 +41,14 @@ Costs() } +.create-or-alter function +with (docstring = 'Gets all invoice details with the latest supported version of the FOCUS schema.', folder = 'InvoiceDetails') +InvoiceDetails() +{ + InvoiceDetails_v1_4() +} + + .create-or-alter function with (docstring = 'Gets all prices with the latest supported version of the FOCUS schema.', folder = 'Prices') Prices() diff --git a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/HubSetup_v1_4.kql b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/HubSetup_v1_4.kql index cabdbdfc5..8b813e44a 100644 --- a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/HubSetup_v1_4.kql +++ b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/HubSetup_v1_4.kql @@ -66,30 +66,30 @@ CommitmentDiscountUsage_v1_4() } -// ContractCommitment_final_v1_4 +// ContractCommitments_final_v1_4 .create-or-alter function -with (docstring = 'Gets all contract commitments aligned to FOCUS 1.4.', folder = 'ContractCommitment') -ContractCommitment_v1_4() +with (docstring = 'Gets all contract commitments aligned to FOCUS 1.4.', folder = 'ContractCommitments') +ContractCommitments_v1_4() { - database('Ingestion').ContractCommitment_final_v1_4 - | union ( - database('Ingestion').ContractCommitment_final_v1_3 - // FOCUS 1.4 column defaults (data was ingested under v1_3 schema) - | extend BenefitCategory = '' - | extend ContractCommitmentApplicability = dynamic(null) - | extend Created = datetime(null) - | extend DiscountPercentage = real(null) - | extend DurationType = '' - | extend FulfillmentInterval = '' - | extend LastUpdated = datetime(null) - | extend LifecycleStatus = '' - | extend Model = '' - | extend OfferCategory = '' - | extend PaymentInterval = '' - | extend PaymentModel = '' - | extend PaymentUpfrontPercentage = real(null) - | extend PricingCurrencyContractCommitmentCost = real(null) - ) + database('Ingestion').ContractCommitments_final_v1_4 +} + + +// BillingPeriods_final_v1_4 +.create-or-alter function +with (docstring = 'Gets all billing periods aligned to FOCUS 1.4.', folder = 'BillingPeriods') +BillingPeriods_v1_4() +{ + database('Ingestion').BillingPeriods_final_v1_4 +} + + +// InvoiceDetails_final_v1_4 +.create-or-alter function +with (docstring = 'Gets all invoice details aligned to FOCUS 1.4.', folder = 'InvoiceDetails') +InvoiceDetails_v1_4() +{ + database('Ingestion').InvoiceDetails_final_v1_4 } @@ -225,33 +225,55 @@ Costs_v1_4() | extend SkuPriceDetails = iff(isempty(SkuPriceDetails.OperatingSystem) and isnotempty(x_SkuOperatingSystem), parse_json(replace_string(tostring(SkuPriceDetails), '}', strcat(@',"OperatingSystem":"', x_SkuOperatingSystem, '"}'))), SkuPriceDetails) - // FOCUS 1.3 column defaults (data was ingested under v1_0 schema) - | extend AllocatedMethodId = '' - | extend AllocatedMethodDetails = dynamic(null) - | extend AllocatedResourceId = '' - | extend AllocatedResourceName = '' - | extend AllocatedTags = dynamic(null) - | extend ContractApplied = dynamic(null) - | extend ServiceProviderName = ProviderName - | extend HostProviderName = PublisherName + // FOCUS 1.4 column defaults (data was ingested under v1_0 schema) + | extend AllocatedMethodId = '' + | extend AllocatedMethodDetails = dynamic(null) + | extend AllocatedResourceId = '' + | extend AllocatedResourceName = '' + | extend AllocatedTags = dynamic(null) + | extend CommitmentProgramEligibilityDetails = dynamic(null) + | extend ContractApplied = dynamic(null) + | extend ContractCommitmentBenefitCategory = '' + | extend ContractCommitmentCreated = datetime(null) + | extend ContractCommitmentDiscountPercentage = real(null) + | extend ContractCommitmentDurationType = '' + | extend ContractCommitmentFulfillmentInterval = '' + | extend ContractCommitmentLastUpdated = datetime(null) + | extend ContractCommitmentLifecycleStatus = '' + | extend ContractCommitmentModel = '' + | extend ContractCommitmentOfferCategory = '' + | extend ContractCommitmentPaymentInterval = '' + | extend ContractCommitmentPaymentModel = '' + | extend ContractCommitmentPaymentUpfrontPercentage = real(null) + | extend HostProviderName = PublisherName + | extend InvoiceDetailId = '' + | extend ServiceProviderName = ProviderName ) | union ( database('Ingestion').Costs_final_v1_2 - // FOCUS 1.3 column defaults (data was ingested under v1_2 schema) - | extend AllocatedMethodId = '' - | extend AllocatedMethodDetails = dynamic(null) - | extend AllocatedResourceId = '' - | extend AllocatedResourceName = '' - | extend AllocatedTags = dynamic(null) - | extend ContractApplied = dynamic(null) - | extend ServiceProviderName = ProviderName - | extend HostProviderName = PublisherName - ) - | union ( - database('Ingestion').Costs_final_v1_3 - // No column adds needed; v1_3 already has the FOCUS 1.3 columns. The - // deprecated ProviderName / PublisherName fall off via the | project - // list below since FOCUS 1.4 removes them. + // FOCUS 1.4 column defaults (data was ingested under v1_2 schema) + | extend AllocatedMethodId = '' + | extend AllocatedMethodDetails = dynamic(null) + | extend AllocatedResourceId = '' + | extend AllocatedResourceName = '' + | extend AllocatedTags = dynamic(null) + | extend CommitmentProgramEligibilityDetails = dynamic(null) + | extend ContractApplied = dynamic(null) + | extend ContractCommitmentBenefitCategory = '' + | extend ContractCommitmentCreated = datetime(null) + | extend ContractCommitmentDiscountPercentage = real(null) + | extend ContractCommitmentDurationType = '' + | extend ContractCommitmentFulfillmentInterval = '' + | extend ContractCommitmentLastUpdated = datetime(null) + | extend ContractCommitmentLifecycleStatus = '' + | extend ContractCommitmentModel = '' + | extend ContractCommitmentOfferCategory = '' + | extend ContractCommitmentPaymentInterval = '' + | extend ContractCommitmentPaymentModel = '' + | extend ContractCommitmentPaymentUpfrontPercentage = real(null) + | extend HostProviderName = PublisherName + | extend InvoiceDetailId = '' + | extend ServiceProviderName = ProviderName ) | extend SkuPriceDetails = iff(isnotempty(SkuPriceDetails), SkuPriceDetails, parse_json(replace_regex(tostring(x_SkuDetails), @'([\{,])"', @'\1"x_'))) | project @@ -283,13 +305,27 @@ Costs_v1_4() CommitmentDiscountStatus, CommitmentDiscountType, CommitmentDiscountUnit, + CommitmentProgramEligibilityDetails, ConsumedQuantity, ConsumedUnit, ContractApplied, + ContractCommitmentBenefitCategory, + ContractCommitmentCreated, + ContractCommitmentDiscountPercentage, + ContractCommitmentDurationType, + ContractCommitmentFulfillmentInterval, + ContractCommitmentLastUpdated, + ContractCommitmentLifecycleStatus, + ContractCommitmentModel, + ContractCommitmentOfferCategory, + ContractCommitmentPaymentInterval, + ContractCommitmentPaymentModel, + ContractCommitmentPaymentUpfrontPercentage, ContractedCost, ContractedUnitPrice, EffectiveCost, HostProviderName, + InvoiceDetailId, InvoiceId, InvoiceIssuerName, ListCost, From 7181c9aa6b062c04b139239df0053adc5999ee77 Mon Sep 17 00:00:00 2001 From: Michael Flanakin Date: Wed, 15 Jul 2026 13:14:25 -0700 Subject: [PATCH 4/4] fix(hubs): address PR #2128 review feedback for FOCUS 1.4 hub functions - Union _final_v1_2 tables into CommitmentDiscountUsage/Prices/Recommendations/Transactions_v1_4() to avoid data loss for existing hubs - Add _final_v1_4 down-convert arms to all 5 dataset functions in HubSetup_v1_0.kql and HubSetup_v1_2.kql for back compat with reports pinned to older schemas - Fix swapped HostProviderName/ServiceProviderName up-converts in Costs_v1_4() to mirror ingestion mapping - Remove duplicated unversioned function tails from HubSetup_v1_4.kql and HubSetup_v1_2.kql (HubSetup_Latest.kql is the single owner) - Repoint dashboard.json from Costs_v1_2 to Costs_v1_4 - Harden CommitmentDiscountUsage_v1_4() Services lookup with take_any to prevent row fan-out - Alphabetize function order, remove stale comment, and fix trailing whitespace in HubSetup_v1_4.kql Co-Authored-By: Claude Fable 5 --- src/templates/finops-hub/dashboard.json | 4 +- .../Analytics/scripts/HubSetup_v1_0.kql | 114 +++++++++++++++++- .../Analytics/scripts/HubSetup_v1_2.kql | 79 ++++++------ .../Analytics/scripts/HubSetup_v1_4.kql | 111 +++++++---------- 4 files changed, 192 insertions(+), 116 deletions(-) diff --git a/src/templates/finops-hub/dashboard.json b/src/templates/finops-hub/dashboard.json index dab4f6b27..c2ea87a78 100644 --- a/src/templates/finops-hub/dashboard.json +++ b/src/templates/finops-hub/dashboard.json @@ -3313,7 +3313,7 @@ { "id": "eb9259cc-05b7-4441-a66d-a29026fe371b", "dataSource": { "kind": "inline", "dataSourceId": "23540be2-ffc9-4b61-8c4c-05e493e682a6" }, - "text": "Costs_v1_2\n//\n// Apply summarization settings\n| where ChargePeriodStart >= monthsago(numberOfMonths)\n| where isempty(selectedBillingCurrency) or BillingCurrency == selectedBillingCurrency\n| as filteredCosts\n| extend x_ChargeMonth = startofmonth(ChargePeriodStart)\n// TODO: Should we add granularity? -- | extend x_ReportingDate = iff(#\"Default Granularity\" == 'Monthly'), x_ChargeMonth, startofday(ChargePeriodStart))\n//\n// SKU details\n| extend x_SkuUsageType = tostring(coalesce(SkuPriceDetails.x_UsageType, x_SkuDetails.UsageType))\n| extend x_SkuLicenseUnusedQuantity = x_SkuLicenseQuantity - x_SkuCoreCount\n//\n// Commitment discounts\n| extend x_CommitmentDiscountKey = iff(isempty(x_SkuInstanceType), '', strcat(x_SkuInstanceType, x_SkuMeterId))\n| extend x_SkuTermLabel = case(isempty(x_SkuTerm) or x_SkuTerm <= 0, '', x_SkuTerm < 12, strcat(x_SkuTerm, ' month', iff(x_SkuTerm != 1, 's', '')), strcat(x_SkuTerm / 12, ' year', iff(x_SkuTerm != 12, 's', '')))\n//\n// CSP partners\n// x_PartnerBilledCredit = iff(x_PartnerCreditApplied, BilledCost * x_PartnerCreditRate, todouble(0))\n// x_PartnerEffectiveCredit = iff(x_PartnerCreditApplied, EffectiveCost * x_PartnerCreditRate, todouble(0))\n//\n// Toolkit\n| extend x_ToolkitTool = tostring(Tags['ftk-tool'])\n| extend x_ToolkitVersion = tostring(Tags['ftk-version'])\n| extend tmp_ResourceParent = database('Ingestion').parse_resourceid(Tags['cm-resource-parent'])\n| extend x_ResourceParentId = tostring(tmp_ResourceParent.ResourceId)\n| extend x_ResourceParentName = tostring(tmp_ResourceParent.ResourceName)\n| extend x_ResourceParentType = tostring(tmp_ResourceParent.ResourceType)\n//\n// TODO: Only add differentiators when the name is not unique\n| extend CommitmentDiscountNameUnique = iff(isempty(CommitmentDiscountId), '', strcat(CommitmentDiscountName, ' (', CommitmentDiscountType, ')'))\n| extend ResourceNameUnique = iff(isempty(ResourceId), '', strcat(ResourceName, ' (', ResourceType, ')'))\n| extend x_ResourceGroupNameUnique = iff(isempty(x_ResourceGroupName), '', strcat(x_ResourceGroupName, ' (', SubAccountName, ')'))\n| extend SubAccountNameUnique = iff(isempty(SubAccountId), '', strcat(SubAccountName, ' (', split(SubAccountId, '/')[3], ')'))\n//\n// Explain why cost is 0\n| extend x_FreeReason = case(\n BilledCost != 0.0 or EffectiveCost != 0.0, '',\n PricingCategory == 'Committed', strcat('Unknown ', CommitmentDiscountStatus, ' Commitment'),\n x_BilledUnitPrice == 0.0 and x_EffectiveUnitPrice == 0.0 and ContractedUnitPrice == 0.0 and ListUnitPrice == 0.0 and isempty(CommitmentDiscountType), case(\n x_SkuDescription contains 'Trial', 'Trial',\n x_SkuDescription contains 'Preview', 'Preview',\n 'Other'\n ),\n x_BilledUnitPrice > 0.0 or x_EffectiveUnitPrice > 0.0, case(\n PricingQuantity > 0.0, 'Low Usage',\n PricingQuantity == 0.0, 'No Usage',\n 'Unknown Negative Quantity'\n ),\n 'Unknown'\n)\n//\n| extend x_ResourceTop1K = ChargeCategory != 'Usage' or isempty(ResourceId) or ResourceId in (\n filteredCosts\n | where isnotempty(ResourceId) and ChargeCategory == 'Usage'\n | summarize sum(EffectiveCost) by ResourceId\n | order by sum_EffectiveCost desc\n | limit 1000\n | distinct ResourceId\n)\n//\n| project-away tmp_ResourceParent", + "text": "Costs_v1_4\n//\n// Apply summarization settings\n| where ChargePeriodStart >= monthsago(numberOfMonths)\n| where isempty(selectedBillingCurrency) or BillingCurrency == selectedBillingCurrency\n| as filteredCosts\n| extend x_ChargeMonth = startofmonth(ChargePeriodStart)\n// TODO: Should we add granularity? -- | extend x_ReportingDate = iff(#\"Default Granularity\" == 'Monthly'), x_ChargeMonth, startofday(ChargePeriodStart))\n//\n// SKU details\n| extend x_SkuUsageType = tostring(coalesce(SkuPriceDetails.x_UsageType, x_SkuDetails.UsageType))\n| extend x_SkuLicenseUnusedQuantity = x_SkuLicenseQuantity - x_SkuCoreCount\n//\n// Commitment discounts\n| extend x_CommitmentDiscountKey = iff(isempty(x_SkuInstanceType), '', strcat(x_SkuInstanceType, x_SkuMeterId))\n| extend x_SkuTermLabel = case(isempty(x_SkuTerm) or x_SkuTerm <= 0, '', x_SkuTerm < 12, strcat(x_SkuTerm, ' month', iff(x_SkuTerm != 1, 's', '')), strcat(x_SkuTerm / 12, ' year', iff(x_SkuTerm != 12, 's', '')))\n//\n// CSP partners\n// x_PartnerBilledCredit = iff(x_PartnerCreditApplied, BilledCost * x_PartnerCreditRate, todouble(0))\n// x_PartnerEffectiveCredit = iff(x_PartnerCreditApplied, EffectiveCost * x_PartnerCreditRate, todouble(0))\n//\n// Toolkit\n| extend x_ToolkitTool = tostring(Tags['ftk-tool'])\n| extend x_ToolkitVersion = tostring(Tags['ftk-version'])\n| extend tmp_ResourceParent = database('Ingestion').parse_resourceid(Tags['cm-resource-parent'])\n| extend x_ResourceParentId = tostring(tmp_ResourceParent.ResourceId)\n| extend x_ResourceParentName = tostring(tmp_ResourceParent.ResourceName)\n| extend x_ResourceParentType = tostring(tmp_ResourceParent.ResourceType)\n//\n// TODO: Only add differentiators when the name is not unique\n| extend CommitmentDiscountNameUnique = iff(isempty(CommitmentDiscountId), '', strcat(CommitmentDiscountName, ' (', CommitmentDiscountType, ')'))\n| extend ResourceNameUnique = iff(isempty(ResourceId), '', strcat(ResourceName, ' (', ResourceType, ')'))\n| extend x_ResourceGroupNameUnique = iff(isempty(x_ResourceGroupName), '', strcat(x_ResourceGroupName, ' (', SubAccountName, ')'))\n| extend SubAccountNameUnique = iff(isempty(SubAccountId), '', strcat(SubAccountName, ' (', split(SubAccountId, '/')[3], ')'))\n//\n// Explain why cost is 0\n| extend x_FreeReason = case(\n BilledCost != 0.0 or EffectiveCost != 0.0, '',\n PricingCategory == 'Committed', strcat('Unknown ', CommitmentDiscountStatus, ' Commitment'),\n x_BilledUnitPrice == 0.0 and x_EffectiveUnitPrice == 0.0 and ContractedUnitPrice == 0.0 and ListUnitPrice == 0.0 and isempty(CommitmentDiscountType), case(\n x_SkuDescription contains 'Trial', 'Trial',\n x_SkuDescription contains 'Preview', 'Preview',\n 'Other'\n ),\n x_BilledUnitPrice > 0.0 or x_EffectiveUnitPrice > 0.0, case(\n PricingQuantity > 0.0, 'Low Usage',\n PricingQuantity == 0.0, 'No Usage',\n 'Unknown Negative Quantity'\n ),\n 'Unknown'\n)\n//\n| extend x_ResourceTop1K = ChargeCategory != 'Usage' or isempty(ResourceId) or ResourceId in (\n filteredCosts\n | where isnotempty(ResourceId) and ChargeCategory == 'Usage'\n | summarize sum(EffectiveCost) by ResourceId\n | order by sum_EffectiveCost desc\n | limit 1000\n | distinct ResourceId\n)\n//\n| project-away tmp_ResourceParent", "usedVariables": ["numberOfMonths", "selectedBillingCurrency"] }, { @@ -3324,7 +3324,7 @@ }, { "dataSource": { "kind": "inline", "dataSourceId": "23540be2-ffc9-4b61-8c4c-05e493e682a6" }, - "text": "let months = toscalar(database('Ingestion').HubSettings | project toint(retention.final.months));\nlet currencies = Costs_v1_2\n | where ChargePeriodStart >= monthsago(iff(isempty(months), 24, months))\n | where isnotempty(BillingCurrency)\n | distinct BillingCurrency;\nlet currencyCount = toscalar(currencies | count);\ncurrencies\n| extend Label = BillingCurrency, _Order = 1\n| union (\n print BillingCurrency = '', Label = '⚠️ All', _Order = 0\n | where currencyCount > 1\n)\n| order by _Order asc, Label asc\n| project BillingCurrency, Label\n", + "text": "let months = toscalar(database('Ingestion').HubSettings | project toint(retention.final.months));\nlet currencies = Costs_v1_4\n | where ChargePeriodStart >= monthsago(iff(isempty(months), 24, months))\n | where isnotempty(BillingCurrency)\n | distinct BillingCurrency;\nlet currencyCount = toscalar(currencies | count);\ncurrencies\n| extend Label = BillingCurrency, _Order = 1\n| union (\n print BillingCurrency = '', Label = '⚠️ All', _Order = 0\n | where currencyCount > 1\n)\n| order by _Order asc, Label asc\n| project BillingCurrency, Label\n", "id": "f2a8c4d6-3b5e-4a7f-9c2d-8e5b1f4a7d9c", "usedVariables": [] } diff --git a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/HubSetup_v1_0.kql b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/HubSetup_v1_0.kql index 2507e9fbf..9d5d08288 100644 --- a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/HubSetup_v1_0.kql +++ b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/HubSetup_v1_0.kql @@ -25,6 +25,16 @@ CommitmentDiscountUsage_v1_0() x_CommitmentDiscountCommittedAmount = todecimal(x_CommitmentDiscountCommittedAmount), x_CommitmentDiscountNormalizedRatio = todecimal(x_CommitmentDiscountNormalizedRatio) ) + | union ( + database('Ingestion').CommitmentDiscountUsage_final_v1_4 + // Convert real to decimal + | extend + CommitmentDiscountQuantity = todecimal(CommitmentDiscountQuantity), + ConsumedQuantity = todecimal(ConsumedQuantity), + x_CommitmentDiscountCommittedCount = todecimal(x_CommitmentDiscountCommittedCount), + x_CommitmentDiscountCommittedAmount = todecimal(x_CommitmentDiscountCommittedAmount), + x_CommitmentDiscountNormalizedRatio = todecimal(x_CommitmentDiscountNormalizedRatio) + ) | project ChargePeriodEnd, ChargePeriodStart, @@ -94,6 +104,65 @@ Costs_v1_0() // Generate historical x_SkuDetails format from SkuPriceDetails | extend x_SkuDetails = iff(isnotempty(x_SkuDetails), x_SkuDetails, parse_json(replace_regex(tostring(SkuPriceDetails), @'([\{,])"x_', @'\1"'))) ) + | union ( + database('Ingestion').Costs_final_v1_4 + // Convert real to decimal + | extend + BilledCost = todecimal(BilledCost), + CommitmentDiscountQuantity = todecimal(CommitmentDiscountQuantity), + ConsumedQuantity = todecimal(ConsumedQuantity), + ContractedCost = todecimal(ContractedCost), + ContractedUnitPrice = todecimal(ContractedUnitPrice), + EffectiveCost = todecimal(EffectiveCost), + ListCost = todecimal(ListCost), + ListUnitPrice = todecimal(ListUnitPrice), + PricingQuantity = todecimal(PricingQuantity), + x_BilledCostInUsd = todecimal(x_BilledCostInUsd), + x_BilledUnitPrice = todecimal(x_BilledUnitPrice), + x_BillingExchangeRate = todecimal(x_BillingExchangeRate), + x_CommitmentDiscountNormalizedRatio = todecimal(x_CommitmentDiscountNormalizedRatio), + x_ContractedCostInUsd = todecimal(x_ContractedCostInUsd), + x_CurrencyConversionRate = todecimal(x_CurrencyConversionRate), + x_EffectiveCostInUsd = todecimal(x_EffectiveCostInUsd), + x_EffectiveUnitPrice = todecimal(x_EffectiveUnitPrice), + x_ListCostInUsd = todecimal(x_ListCostInUsd), + x_PricingBlockSize = todecimal(x_PricingBlockSize) + // Rename columns + | project-rename + x_InvoiceId = InvoiceId, + x_PricingCurrency = PricingCurrency, + x_SkuMeterName = SkuMeter + // Generate historical x_SkuDetails format from SkuPriceDetails + | extend x_SkuDetails = iff(isnotempty(x_SkuDetails), x_SkuDetails, parse_json(replace_regex(tostring(SkuPriceDetails), @'([\{,])"x_', @'\1"'))) + // Restore deprecated columns removed in FOCUS 1.4 + | extend + ProviderName = HostProviderName, + PublisherName = ServiceProviderName + // Remove FOCUS 1.3 and 1.4 columns + | project-away + AllocatedMethodDetails, + AllocatedMethodId, + AllocatedResourceId, + AllocatedResourceName, + AllocatedTags, + CommitmentProgramEligibilityDetails, + ContractApplied, + ContractCommitmentBenefitCategory, + ContractCommitmentCreated, + ContractCommitmentDiscountPercentage, + ContractCommitmentDurationType, + ContractCommitmentFulfillmentInterval, + ContractCommitmentLastUpdated, + ContractCommitmentLifecycleStatus, + ContractCommitmentModel, + ContractCommitmentOfferCategory, + ContractCommitmentPaymentInterval, + ContractCommitmentPaymentModel, + ContractCommitmentPaymentUpfrontPercentage, + HostProviderName, + InvoiceDetailId, + ServiceProviderName + ) | project AvailabilityZone, BilledCost, @@ -238,7 +307,30 @@ Prices_v1_0() x_SkuIncludedQuantity = todecimal(x_SkuIncludedQuantity), x_SkuTier = todecimal(x_SkuTier), x_TotalUnitPriceDiscount = todecimal(x_TotalUnitPriceDiscount), - x_TotalUnitPriceDiscountPercent = todecimal(x_TotalUnitPriceDiscountPercent) + x_TotalUnitPriceDiscountPercent = todecimal(x_TotalUnitPriceDiscountPercent) + // Rename columns + | project-rename + x_PricingCurrency = PricingCurrency, + x_SkuMeterName = SkuMeter + ) + | union ( + database('Ingestion').Prices_final_v1_4 + // Convert real to decimal + | extend + ContractedUnitPrice = todecimal(ContractedUnitPrice), + ListUnitPrice = todecimal(ListUnitPrice), + x_BaseUnitPrice = todecimal(x_BaseUnitPrice), + x_CommitmentDiscountNormalizedRatio = todecimal(x_CommitmentDiscountNormalizedRatio), + x_ContractedUnitPriceDiscount = todecimal(x_ContractedUnitPriceDiscount), + x_ContractedUnitPriceDiscountPercent = todecimal(x_ContractedUnitPriceDiscountPercent), + x_EffectiveUnitPrice = todecimal(x_EffectiveUnitPrice), + x_EffectiveUnitPriceDiscount = todecimal(x_EffectiveUnitPriceDiscount), + x_EffectiveUnitPriceDiscountPercent = todecimal(x_EffectiveUnitPriceDiscountPercent), + x_PricingBlockSize = todecimal(x_PricingBlockSize), + x_SkuIncludedQuantity = todecimal(x_SkuIncludedQuantity), + x_SkuTier = todecimal(x_SkuTier), + x_TotalUnitPriceDiscount = todecimal(x_TotalUnitPriceDiscount), + x_TotalUnitPriceDiscountPercent = todecimal(x_TotalUnitPriceDiscountPercent) // Rename columns | project-rename x_PricingCurrency = PricingCurrency, @@ -315,6 +407,14 @@ Recommendations_v1_0() x_EffectiveCostBefore = todecimal(x_EffectiveCostBefore), x_EffectiveCostSavings = todecimal(x_EffectiveCostSavings) ) + | union ( + database('Ingestion').Recommendations_final_v1_4 + // Convert real to decimal + | extend + x_EffectiveCostAfter = todecimal(x_EffectiveCostAfter), + x_EffectiveCostBefore = todecimal(x_EffectiveCostBefore), + x_EffectiveCostSavings = todecimal(x_EffectiveCostSavings) + ) | project ProviderName, SubAccountId, @@ -349,6 +449,18 @@ Transactions_v1_0() | project-rename x_InvoiceId = InvoiceId ) + | union ( + database('Ingestion').Transactions_final_v1_4 + // Convert real to decimal + | extend + BilledCost = todecimal(BilledCost), + PricingQuantity = todecimal(PricingQuantity), + x_MonetaryCommitment = todecimal(x_MonetaryCommitment), + x_Overage = todecimal(x_Overage) + // Rename columns + | project-rename + x_InvoiceId = InvoiceId + ) | project BilledCost, BillingAccountId, diff --git a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/HubSetup_v1_2.kql b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/HubSetup_v1_2.kql index ffc8d5de5..a5a8baa4f 100644 --- a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/HubSetup_v1_2.kql +++ b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/HubSetup_v1_2.kql @@ -15,6 +15,7 @@ with (docstring = 'Gets all commitment discount usage records aligned to FOCUS 1 CommitmentDiscountUsage_v1_2() { database('Ingestion').CommitmentDiscountUsage_final_v1_2 + | union (database('Ingestion').CommitmentDiscountUsage_final_v1_4) | union ( database('Ingestion').CommitmentDiscountUsage_final_v1_0 // Convert decimal to real @@ -199,6 +200,37 @@ Costs_v1_2() parse_json(replace_string(tostring(SkuPriceDetails), '}', strcat(@',"OperatingSystem":"', x_SkuOperatingSystem, '"}'))), SkuPriceDetails) ) + | union ( + database('Ingestion').Costs_final_v1_4 + // Restore deprecated columns removed in FOCUS 1.4 + | extend + ProviderName = HostProviderName, + PublisherName = ServiceProviderName + // Remove FOCUS 1.3 and 1.4 columns + | project-away + AllocatedMethodDetails, + AllocatedMethodId, + AllocatedResourceId, + AllocatedResourceName, + AllocatedTags, + CommitmentProgramEligibilityDetails, + ContractApplied, + ContractCommitmentBenefitCategory, + ContractCommitmentCreated, + ContractCommitmentDiscountPercentage, + ContractCommitmentDurationType, + ContractCommitmentFulfillmentInterval, + ContractCommitmentLastUpdated, + ContractCommitmentLifecycleStatus, + ContractCommitmentModel, + ContractCommitmentOfferCategory, + ContractCommitmentPaymentInterval, + ContractCommitmentPaymentModel, + ContractCommitmentPaymentUpfrontPercentage, + HostProviderName, + InvoiceDetailId, + ServiceProviderName + ) | extend SkuPriceDetails = iff(isnotempty(SkuPriceDetails), SkuPriceDetails, parse_json(replace_regex(tostring(x_SkuDetails), @'([\{,])"', @'\1"x_'))) | project AvailabilityZone, @@ -365,6 +397,7 @@ with (docstring = 'Gets all prices aligned to FOCUS 1.2.', folder = 'Prices') Prices_v1_2() { database('Ingestion').Prices_final_v1_2 + | union (database('Ingestion').Prices_final_v1_4) | union ( database('Ingestion').Prices_final_v1_0 // Convert decimal to real @@ -452,6 +485,7 @@ with (docstring = 'Gets all recommendations aligned to FOCUS 1.2.', folder = 'Re Recommendations_v1_2() { database('Ingestion').Recommendations_final_v1_2 + | union (database('Ingestion').Recommendations_final_v1_4) | union ( database('Ingestion').Recommendations_final_v1_0 // Convert decimal to real @@ -490,6 +524,7 @@ with (docstring = 'Gets all transactions aligned to FOCUS 1.2.', folder = 'Trans Transactions_v1_2() { database('Ingestion').Transactions_final_v1_2 + | union (database('Ingestion').Transactions_final_v1_4) | union ( database('Ingestion').Transactions_final_v1_0 // Convert decimal to real @@ -543,47 +578,3 @@ Transactions_v1_2() x_SubscriptionId, x_TransactionType } - - -//====================================================================================================================== -// Latest FOCUS version -//====================================================================================================================== - -.create-or-alter function -with (docstring = 'Gets all commitment discount usage records with the latest supported version of the FOCUS schema.', folder = 'CommitmentDiscountUsage') -CommitmentDiscountUsage() -{ - CommitmentDiscountUsage_v1_2() -} - - -.create-or-alter function -with (docstring = 'Gets all cost and usage records with the latest supported version of the FOCUS schema.', folder = 'Costs') -Costs() -{ - Costs_v1_2() -} - - -.create-or-alter function -with (docstring = 'Gets all prices with the latest supported version of the FOCUS schema.', folder = 'Prices') -Prices() -{ - Prices_v1_2() -} - - -.create-or-alter function -with (docstring = 'Gets all recommendations with the latest supported version of the FOCUS schema.', folder = 'Recommendations') -Recommendations() -{ - Recommendations_v1_2() -} - - -.create-or-alter function -with (docstring = 'Gets all transactions with the latest supported version of the FOCUS schema.', folder = 'Transactions') -Transactions() -{ - Transactions_v1_2() -} diff --git a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/HubSetup_v1_4.kql b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/HubSetup_v1_4.kql index 8b813e44a..04b655082 100644 --- a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/HubSetup_v1_4.kql +++ b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/HubSetup_v1_4.kql @@ -9,12 +9,22 @@ // For allowed commands, see https://learn.microsoft.com/azure/data-explorer/database-script +// BillingPeriods_final_v1_4 +.create-or-alter function +with (docstring = 'Gets all billing periods aligned to FOCUS 1.4.', folder = 'BillingPeriods') +BillingPeriods_v1_4() +{ + database('Ingestion').BillingPeriods_final_v1_4 +} + + // CommitmentDiscountUsage_final_v1_4 .create-or-alter function with (docstring = 'Gets all commitment discount usage records aligned to FOCUS 1.4.', folder = 'CommitmentDiscountUsage') CommitmentDiscountUsage_v1_4() { database('Ingestion').CommitmentDiscountUsage_final_v1_4 + | union (database('Ingestion').CommitmentDiscountUsage_final_v1_2) | union ( database('Ingestion').CommitmentDiscountUsage_final_v1_0 // Convert decimal to real @@ -24,7 +34,7 @@ CommitmentDiscountUsage_v1_4() x_CommitmentDiscountCommittedAmount = toreal(x_CommitmentDiscountCommittedAmount), x_CommitmentDiscountNormalizedRatio = toreal(x_CommitmentDiscountNormalizedRatio) // Add new columns - | lookup kind=leftouter (Services | distinct x_ResourceType, ServiceSubcategory) on x_ResourceType + | lookup kind=leftouter (Services | where isnotempty(x_ResourceType) | summarize take_any(ServiceSubcategory) by x_ResourceType) on x_ResourceType | extend CommitmentDiscountQuantity = ConsumedQuantity * x_CommitmentDiscountNormalizedRatio | extend CommitmentDiscountUnit = case( x_CommitmentDiscountNormalizedRatio == 1, 'Hours', @@ -75,24 +85,6 @@ ContractCommitments_v1_4() } -// BillingPeriods_final_v1_4 -.create-or-alter function -with (docstring = 'Gets all billing periods aligned to FOCUS 1.4.', folder = 'BillingPeriods') -BillingPeriods_v1_4() -{ - database('Ingestion').BillingPeriods_final_v1_4 -} - - -// InvoiceDetails_final_v1_4 -.create-or-alter function -with (docstring = 'Gets all invoice details aligned to FOCUS 1.4.', folder = 'InvoiceDetails') -InvoiceDetails_v1_4() -{ - database('Ingestion').InvoiceDetails_final_v1_4 -} - - // Costs_final_v1_4 .create-or-alter function with (docstring = 'Gets all cost and usage records aligned to FOCUS 1.4.', folder = 'Costs') @@ -245,9 +237,16 @@ Costs_v1_4() | extend ContractCommitmentPaymentInterval = '' | extend ContractCommitmentPaymentModel = '' | extend ContractCommitmentPaymentUpfrontPercentage = real(null) - | extend HostProviderName = PublisherName + | extend HostProviderName = ProviderName | extend InvoiceDetailId = '' - | extend ServiceProviderName = ProviderName + | extend ServiceProviderName = case( + // Use the Marketplace publisher when available (normalized to the cloud provider for non-Marketplace charges) + isnotempty(PublisherName), PublisherName, + // Fall back to the deprecated ProviderName when PublisherName could not be resolved + isnotempty(ProviderName), ProviderName, + // Must not be null per FOCUS; Cost Management data is always Microsoft + 'Microsoft' + ) ) | union ( database('Ingestion').Costs_final_v1_2 @@ -271,9 +270,16 @@ Costs_v1_4() | extend ContractCommitmentPaymentInterval = '' | extend ContractCommitmentPaymentModel = '' | extend ContractCommitmentPaymentUpfrontPercentage = real(null) - | extend HostProviderName = PublisherName + | extend HostProviderName = ProviderName | extend InvoiceDetailId = '' - | extend ServiceProviderName = ProviderName + | extend ServiceProviderName = case( + // Use the Marketplace publisher when available (normalized to the cloud provider for non-Marketplace charges) + isnotempty(PublisherName), PublisherName, + // Fall back to the deprecated ProviderName when PublisherName could not be resolved + isnotempty(ProviderName), ProviderName, + // Must not be null per FOCUS; Cost Management data is always Microsoft + 'Microsoft' + ) ) | extend SkuPriceDetails = iff(isnotempty(SkuPriceDetails), SkuPriceDetails, parse_json(replace_regex(tostring(x_SkuDetails), @'([\{,])"', @'\1"x_'))) | project @@ -334,7 +340,6 @@ Costs_v1_4() PricingCurrency, PricingQuantity, PricingUnit, - // ProviderName and PublisherName removed in FOCUS 1.4. Use ServiceProviderName / HostProviderName. RegionId, RegionName, ResourceId, @@ -456,12 +461,22 @@ Costs_v1_4() } +// InvoiceDetails_final_v1_4 +.create-or-alter function +with (docstring = 'Gets all invoice details aligned to FOCUS 1.4.', folder = 'InvoiceDetails') +InvoiceDetails_v1_4() +{ + database('Ingestion').InvoiceDetails_final_v1_4 +} + + // Prices_final_v1_4 .create-or-alter function with (docstring = 'Gets all prices aligned to FOCUS 1.4.', folder = 'Prices') Prices_v1_4() { database('Ingestion').Prices_final_v1_4 + | union (database('Ingestion').Prices_final_v1_2) | union ( database('Ingestion').Prices_final_v1_0 // Convert decimal to real @@ -478,7 +493,7 @@ Prices_v1_4() x_SkuIncludedQuantity = toreal(x_SkuIncludedQuantity), x_SkuTier = toreal(x_SkuTier), x_TotalUnitPriceDiscount = toreal(x_TotalUnitPriceDiscount), - x_TotalUnitPriceDiscountPercent = toreal(x_TotalUnitPriceDiscountPercent) + x_TotalUnitPriceDiscountPercent = toreal(x_TotalUnitPriceDiscountPercent) // Rename columns | project-rename PricingCurrency = x_PricingCurrency, @@ -549,6 +564,7 @@ with (docstring = 'Gets all recommendations aligned to FOCUS 1.4.', folder = 'Re Recommendations_v1_4() { database('Ingestion').Recommendations_final_v1_4 + | union (database('Ingestion').Recommendations_final_v1_2) | union ( database('Ingestion').Recommendations_final_v1_0 // Convert decimal to real @@ -587,6 +603,7 @@ with (docstring = 'Gets all transactions aligned to FOCUS 1.4.', folder = 'Trans Transactions_v1_4() { database('Ingestion').Transactions_final_v1_4 + | union (database('Ingestion').Transactions_final_v1_2) | union ( database('Ingestion').Transactions_final_v1_0 // Convert decimal to real @@ -640,47 +657,3 @@ Transactions_v1_4() x_SubscriptionId, x_TransactionType } - - -//====================================================================================================================== -// Latest FOCUS version -//====================================================================================================================== - -.create-or-alter function -with (docstring = 'Gets all commitment discount usage records with the latest supported version of the FOCUS schema.', folder = 'CommitmentDiscountUsage') -CommitmentDiscountUsage() -{ - CommitmentDiscountUsage_v1_4() -} - - -.create-or-alter function -with (docstring = 'Gets all cost and usage records with the latest supported version of the FOCUS schema.', folder = 'Costs') -Costs() -{ - Costs_v1_4() -} - - -.create-or-alter function -with (docstring = 'Gets all prices with the latest supported version of the FOCUS schema.', folder = 'Prices') -Prices() -{ - Prices_v1_4() -} - - -.create-or-alter function -with (docstring = 'Gets all recommendations with the latest supported version of the FOCUS schema.', folder = 'Recommendations') -Recommendations() -{ - Recommendations_v1_4() -} - - -.create-or-alter function -with (docstring = 'Gets all transactions with the latest supported version of the FOCUS schema.', folder = 'Transactions') -Transactions() -{ - Transactions_v1_4() -}