Add FOCUS 1.4 hub functions (HubSetup_v1_4.kql)

src/templates/finops-hub/.build.config

@@ -45,6 +45,7 @@
         "modules/Microsoft.FinOpsHubs/Analytics/scripts/HubSetup_OpenData.kql",
         "modules/Microsoft.FinOpsHubs/Analytics/scripts/HubSetup_v1_0.kql",
         "modules/Microsoft.FinOpsHubs/Analytics/scripts/HubSetup_v1_2.kql",
+        "modules/Microsoft.FinOpsHubs/Analytics/scripts/HubSetup_v1_4.kql",
         "modules/Microsoft.FinOpsHubs/Analytics/scripts/HubSetup_Latest.kql"
       ]
     }

src/templates/finops-hub/dashboard.json

@@ -3313,7 +3313,7 @@
     {
       "id": "eb9259cc-05b7-4441-a66d-a29026fe371b",
       "dataSource": { "kind": "inline", "dataSourceId": "23540be2-ffc9-4b61-8c4c-05e493e682a6" },
-      "text": "Costs_v1_2\n//\n// Apply summarization settings\n| where ChargePeriodStart >= monthsago(numberOfMonths)\n| where isempty(selectedBillingCurrency) or BillingCurrency == selectedBillingCurrency\n| as filteredCosts\n| extend x_ChargeMonth = startofmonth(ChargePeriodStart)\n// TODO: Should we add granularity? -- | extend x_ReportingDate = iff(#\"Default Granularity\" == 'Monthly'), x_ChargeMonth, startofday(ChargePeriodStart))\n//\n// SKU details\n| extend x_SkuUsageType = tostring(coalesce(SkuPriceDetails.x_UsageType, x_SkuDetails.UsageType))\n| extend x_SkuLicenseUnusedQuantity = x_SkuLicenseQuantity - x_SkuCoreCount\n//\n// Commitment discounts\n| extend x_CommitmentDiscountKey = iff(isempty(x_SkuInstanceType), '', strcat(x_SkuInstanceType, x_SkuMeterId))\n| extend x_SkuTermLabel = case(isempty(x_SkuTerm) or x_SkuTerm <= 0, '', x_SkuTerm < 12, strcat(x_SkuTerm, ' month', iff(x_SkuTerm != 1, 's', '')), strcat(x_SkuTerm / 12, ' year', iff(x_SkuTerm != 12, 's', '')))\n//\n// CSP partners\n// x_PartnerBilledCredit = iff(x_PartnerCreditApplied, BilledCost * x_PartnerCreditRate, todouble(0))\n// x_PartnerEffectiveCredit = iff(x_PartnerCreditApplied, EffectiveCost * x_PartnerCreditRate, todouble(0))\n//\n// Toolkit\n| extend x_ToolkitTool = tostring(Tags['ftk-tool'])\n| extend x_ToolkitVersion = tostring(Tags['ftk-version'])\n| extend tmp_ResourceParent = database('Ingestion').parse_resourceid(Tags['cm-resource-parent'])\n| extend x_ResourceParentId = tostring(tmp_ResourceParent.ResourceId)\n| extend x_ResourceParentName = tostring(tmp_ResourceParent.ResourceName)\n| extend x_ResourceParentType = tostring(tmp_ResourceParent.ResourceType)\n//\n// TODO: Only add differentiators when the name is not unique\n| extend CommitmentDiscountNameUnique = iff(isempty(CommitmentDiscountId), '', strcat(CommitmentDiscountName, ' (', CommitmentDiscountType, ')'))\n| extend ResourceNameUnique           = iff(isempty(ResourceId),           '', strcat(ResourceName,           ' (', ResourceType, ')'))\n| extend x_ResourceGroupNameUnique    = iff(isempty(x_ResourceGroupName),  '', strcat(x_ResourceGroupName,    ' (', SubAccountName, ')'))\n| extend SubAccountNameUnique         = iff(isempty(SubAccountId),         '', strcat(SubAccountName,         ' (', split(SubAccountId, '/')[3], ')'))\n//\n// Explain why cost is 0\n| extend x_FreeReason = case(\n    BilledCost != 0.0 or EffectiveCost != 0.0, '',\n    PricingCategory == 'Committed', strcat('Unknown ', CommitmentDiscountStatus, ' Commitment'),\n    x_BilledUnitPrice == 0.0 and x_EffectiveUnitPrice == 0.0 and ContractedUnitPrice == 0.0 and ListUnitPrice == 0.0 and isempty(CommitmentDiscountType), case(\n        x_SkuDescription contains 'Trial', 'Trial',\n        x_SkuDescription contains 'Preview', 'Preview',\n        'Other'\n    ),\n    x_BilledUnitPrice > 0.0 or x_EffectiveUnitPrice > 0.0, case(\n        PricingQuantity > 0.0, 'Low Usage',\n        PricingQuantity == 0.0, 'No Usage',\n        'Unknown Negative Quantity'\n    ),\n    'Unknown'\n)\n//\n| extend x_ResourceTop1K = ChargeCategory != 'Usage' or isempty(ResourceId) or ResourceId in (\n    filteredCosts\n    | where isnotempty(ResourceId) and ChargeCategory == 'Usage'\n    | summarize sum(EffectiveCost) by ResourceId\n    | order by sum_EffectiveCost desc\n    | limit 1000\n    | distinct ResourceId\n)\n//\n| project-away tmp_ResourceParent",
+      "text": "Costs_v1_4\n//\n// Apply summarization settings\n| where ChargePeriodStart >= monthsago(numberOfMonths)\n| where isempty(selectedBillingCurrency) or BillingCurrency == selectedBillingCurrency\n| as filteredCosts\n| extend x_ChargeMonth = startofmonth(ChargePeriodStart)\n// TODO: Should we add granularity? -- | extend x_ReportingDate = iff(#\"Default Granularity\" == 'Monthly'), x_ChargeMonth, startofday(ChargePeriodStart))\n//\n// SKU details\n| extend x_SkuUsageType = tostring(coalesce(SkuPriceDetails.x_UsageType, x_SkuDetails.UsageType))\n| extend x_SkuLicenseUnusedQuantity = x_SkuLicenseQuantity - x_SkuCoreCount\n//\n// Commitment discounts\n| extend x_CommitmentDiscountKey = iff(isempty(x_SkuInstanceType), '', strcat(x_SkuInstanceType, x_SkuMeterId))\n| extend x_SkuTermLabel = case(isempty(x_SkuTerm) or x_SkuTerm <= 0, '', x_SkuTerm < 12, strcat(x_SkuTerm, ' month', iff(x_SkuTerm != 1, 's', '')), strcat(x_SkuTerm / 12, ' year', iff(x_SkuTerm != 12, 's', '')))\n//\n// CSP partners\n// x_PartnerBilledCredit = iff(x_PartnerCreditApplied, BilledCost * x_PartnerCreditRate, todouble(0))\n// x_PartnerEffectiveCredit = iff(x_PartnerCreditApplied, EffectiveCost * x_PartnerCreditRate, todouble(0))\n//\n// Toolkit\n| extend x_ToolkitTool = tostring(Tags['ftk-tool'])\n| extend x_ToolkitVersion = tostring(Tags['ftk-version'])\n| extend tmp_ResourceParent = database('Ingestion').parse_resourceid(Tags['cm-resource-parent'])\n| extend x_ResourceParentId = tostring(tmp_ResourceParent.ResourceId)\n| extend x_ResourceParentName = tostring(tmp_ResourceParent.ResourceName)\n| extend x_ResourceParentType = tostring(tmp_ResourceParent.ResourceType)\n//\n// TODO: Only add differentiators when the name is not unique\n| extend CommitmentDiscountNameUnique = iff(isempty(CommitmentDiscountId), '', strcat(CommitmentDiscountName, ' (', CommitmentDiscountType, ')'))\n| extend ResourceNameUnique           = iff(isempty(ResourceId),           '', strcat(ResourceName,           ' (', ResourceType, ')'))\n| extend x_ResourceGroupNameUnique    = iff(isempty(x_ResourceGroupName),  '', strcat(x_ResourceGroupName,    ' (', SubAccountName, ')'))\n| extend SubAccountNameUnique         = iff(isempty(SubAccountId),         '', strcat(SubAccountName,         ' (', split(SubAccountId, '/')[3], ')'))\n//\n// Explain why cost is 0\n| extend x_FreeReason = case(\n    BilledCost != 0.0 or EffectiveCost != 0.0, '',\n    PricingCategory == 'Committed', strcat('Unknown ', CommitmentDiscountStatus, ' Commitment'),\n    x_BilledUnitPrice == 0.0 and x_EffectiveUnitPrice == 0.0 and ContractedUnitPrice == 0.0 and ListUnitPrice == 0.0 and isempty(CommitmentDiscountType), case(\n        x_SkuDescription contains 'Trial', 'Trial',\n        x_SkuDescription contains 'Preview', 'Preview',\n        'Other'\n    ),\n    x_BilledUnitPrice > 0.0 or x_EffectiveUnitPrice > 0.0, case(\n        PricingQuantity > 0.0, 'Low Usage',\n        PricingQuantity == 0.0, 'No Usage',\n        'Unknown Negative Quantity'\n    ),\n    'Unknown'\n)\n//\n| extend x_ResourceTop1K = ChargeCategory != 'Usage' or isempty(ResourceId) or ResourceId in (\n    filteredCosts\n    | where isnotempty(ResourceId) and ChargeCategory == 'Usage'\n    | summarize sum(EffectiveCost) by ResourceId\n    | order by sum_EffectiveCost desc\n    | limit 1000\n    | distinct ResourceId\n)\n//\n| project-away tmp_ResourceParent",
       "usedVariables": ["numberOfMonths", "selectedBillingCurrency"]
     },
     {
@@ -3324,7 +3324,7 @@
     },
     {
       "dataSource": { "kind": "inline", "dataSourceId": "23540be2-ffc9-4b61-8c4c-05e493e682a6" },
-      "text": "let months = toscalar(database('Ingestion').HubSettings | project toint(retention.final.months));\nlet currencies = Costs_v1_2\n    | where ChargePeriodStart >= monthsago(iff(isempty(months), 24, months))\n    | where isnotempty(BillingCurrency)\n    | distinct BillingCurrency;\nlet currencyCount = toscalar(currencies | count);\ncurrencies\n| extend Label = BillingCurrency, _Order = 1\n| union (\n    print BillingCurrency = '', Label = '⚠️ All', _Order = 0\n    | where currencyCount > 1\n)\n| order by _Order asc, Label asc\n| project BillingCurrency, Label\n",
+      "text": "let months = toscalar(database('Ingestion').HubSettings | project toint(retention.final.months));\nlet currencies = Costs_v1_4\n    | where ChargePeriodStart >= monthsago(iff(isempty(months), 24, months))\n    | where isnotempty(BillingCurrency)\n    | distinct BillingCurrency;\nlet currencyCount = toscalar(currencies | count);\ncurrencies\n| extend Label = BillingCurrency, _Order = 1\n| union (\n    print BillingCurrency = '', Label = '⚠️ All', _Order = 0\n    | where currencyCount > 1\n)\n| order by _Order asc, Label asc\n| project BillingCurrency, Label\n",
       "id": "f2a8c4d6-3b5e-4a7f-9c2d-8e5b1f4a7d9c",
       "usedVariables": []
     }

src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/app.bicep

@@ -421,6 +421,7 @@ module hub_VersionedScripts '../../fx/hub-database.bicep' = if (useAzure) {
     scripts: {
       v1_0: loadTextContent('scripts/HubSetup_v1_0.kql')
       v1_2: loadTextContent('scripts/HubSetup_v1_2.kql')
+      v1_4: loadTextContent('scripts/HubSetup_v1_4.kql')
     }
     continueOnErrors: continueOnErrors
     forceUpdateTag: forceUpdateTag

src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/HubSetup_Latest.kql

@@ -9,41 +9,65 @@
 // For allowed commands, see https://learn.microsoft.com/azure/data-explorer/database-script
 
 
+.create-or-alter function
+with (docstring = 'Gets all billing periods with the latest supported version of the FOCUS schema.', folder = 'BillingPeriods')
+BillingPeriods()
+{
+    BillingPeriods_v1_4()
+}
+
+
 .create-or-alter function
 with (docstring = 'Gets all commitment discount usage records with the latest supported version of the FOCUS schema.', folder = 'CommitmentDiscountUsage')
 CommitmentDiscountUsage()
 {
-    CommitmentDiscountUsage_v1_2()
+    CommitmentDiscountUsage_v1_4()
+}
+
+
+.create-or-alter function
+with (docstring = 'Gets all contract commitments with the latest supported version of the FOCUS schema.', folder = 'ContractCommitments')
+ContractCommitments()
+{
+    ContractCommitments_v1_4()
 }
 
 
 .create-or-alter function
 with (docstring = 'Gets all cost and usage records with the latest supported version of the FOCUS schema.', folder = 'Costs')
 Costs()
 {
-    Costs_v1_2()
+    Costs_v1_4()
+}
+
+
+.create-or-alter function
+with (docstring = 'Gets all invoice details with the latest supported version of the FOCUS schema.', folder = 'InvoiceDetails')
+InvoiceDetails()
+{
+    InvoiceDetails_v1_4()
 }
 
 
 .create-or-alter function
 with (docstring = 'Gets all prices with the latest supported version of the FOCUS schema.', folder = 'Prices')
 Prices()
 {
-    Prices_v1_2()
+    Prices_v1_4()
 }
 
 
 .create-or-alter function
 with (docstring = 'Gets all recommendations with the latest supported version of the FOCUS schema.', folder = 'Recommendations')
 Recommendations()
 {
-    Recommendations_v1_2()
+    Recommendations_v1_4()
 }
 
 
 .create-or-alter function
 with (docstring = 'Gets all transactions with the latest supported version of the FOCUS schema.', folder = 'Transactions')
 Transactions()
 {
-    Transactions_v1_2()
+    Transactions_v1_4()
 }

src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/HubSetup_v1_0.kql

@@ -25,6 +25,16 @@ CommitmentDiscountUsage_v1_0()
             x_CommitmentDiscountCommittedAmount = todecimal(x_CommitmentDiscountCommittedAmount),
             x_CommitmentDiscountNormalizedRatio = todecimal(x_CommitmentDiscountNormalizedRatio)
     )
+    | union (
+        database('Ingestion').CommitmentDiscountUsage_final_v1_4
+        // Convert real to decimal
+        | extend
+            CommitmentDiscountQuantity          = todecimal(CommitmentDiscountQuantity),
+            ConsumedQuantity                    = todecimal(ConsumedQuantity),
+            x_CommitmentDiscountCommittedCount  = todecimal(x_CommitmentDiscountCommittedCount),
+            x_CommitmentDiscountCommittedAmount = todecimal(x_CommitmentDiscountCommittedAmount),
+            x_CommitmentDiscountNormalizedRatio = todecimal(x_CommitmentDiscountNormalizedRatio)
+    )
     | project
         ChargePeriodEnd,
         ChargePeriodStart,
@@ -94,6 +104,65 @@ Costs_v1_0()
         // Generate historical x_SkuDetails format from SkuPriceDetails
         | extend x_SkuDetails = iff(isnotempty(x_SkuDetails), x_SkuDetails, parse_json(replace_regex(tostring(SkuPriceDetails), @'([\{,])"x_', @'\1"')))
     )
+    | union (
+        database('Ingestion').Costs_final_v1_4
+        // Convert real to decimal
+        | extend
+            BilledCost                          = todecimal(BilledCost),
+            CommitmentDiscountQuantity          = todecimal(CommitmentDiscountQuantity),
+            ConsumedQuantity                    = todecimal(ConsumedQuantity),
+            ContractedCost                      = todecimal(ContractedCost),
+            ContractedUnitPrice                 = todecimal(ContractedUnitPrice),
+            EffectiveCost                       = todecimal(EffectiveCost),
+            ListCost                            = todecimal(ListCost),
+            ListUnitPrice                       = todecimal(ListUnitPrice),
+            PricingQuantity                     = todecimal(PricingQuantity),
+            x_BilledCostInUsd                   = todecimal(x_BilledCostInUsd),
+            x_BilledUnitPrice                   = todecimal(x_BilledUnitPrice),
+            x_BillingExchangeRate               = todecimal(x_BillingExchangeRate),
+            x_CommitmentDiscountNormalizedRatio = todecimal(x_CommitmentDiscountNormalizedRatio),
+            x_ContractedCostInUsd               = todecimal(x_ContractedCostInUsd),
+            x_CurrencyConversionRate            = todecimal(x_CurrencyConversionRate),
+            x_EffectiveCostInUsd                = todecimal(x_EffectiveCostInUsd),
+            x_EffectiveUnitPrice                = todecimal(x_EffectiveUnitPrice),
+            x_ListCostInUsd                     = todecimal(x_ListCostInUsd),
+            x_PricingBlockSize                  = todecimal(x_PricingBlockSize)
+        // Rename columns
+        | project-rename
+            x_InvoiceId = InvoiceId,
+            x_PricingCurrency = PricingCurrency,
+            x_SkuMeterName = SkuMeter
+        // Generate historical x_SkuDetails format from SkuPriceDetails
+        | extend x_SkuDetails = iff(isnotempty(x_SkuDetails), x_SkuDetails, parse_json(replace_regex(tostring(SkuPriceDetails), @'([\{,])"x_', @'\1"')))
+        // Restore deprecated columns removed in FOCUS 1.4
+        | extend
+            ProviderName  = HostProviderName,
+            PublisherName = ServiceProviderName
+        // Remove FOCUS 1.3 and 1.4 columns
+        | project-away
+            AllocatedMethodDetails,
+            AllocatedMethodId,
+            AllocatedResourceId,
+            AllocatedResourceName,
+            AllocatedTags,
+            CommitmentProgramEligibilityDetails,
+            ContractApplied,
+            ContractCommitmentBenefitCategory,
+            ContractCommitmentCreated,
+            ContractCommitmentDiscountPercentage,
+            ContractCommitmentDurationType,
+            ContractCommitmentFulfillmentInterval,
+            ContractCommitmentLastUpdated,
+            ContractCommitmentLifecycleStatus,
+            ContractCommitmentModel,
+            ContractCommitmentOfferCategory,
+            ContractCommitmentPaymentInterval,
+            ContractCommitmentPaymentModel,
+            ContractCommitmentPaymentUpfrontPercentage,
+            HostProviderName,
+            InvoiceDetailId,
+            ServiceProviderName
+    )
     | project
         AvailabilityZone,
         BilledCost,
@@ -238,7 +307,30 @@ Prices_v1_0()
             x_SkuIncludedQuantity                = todecimal(x_SkuIncludedQuantity),
             x_SkuTier                            = todecimal(x_SkuTier),
             x_TotalUnitPriceDiscount             = todecimal(x_TotalUnitPriceDiscount),
-            x_TotalUnitPriceDiscountPercent      = todecimal(x_TotalUnitPriceDiscountPercent) 
+            x_TotalUnitPriceDiscountPercent      = todecimal(x_TotalUnitPriceDiscountPercent)
+        // Rename columns
+        | project-rename
+            x_PricingCurrency = PricingCurrency,
+            x_SkuMeterName = SkuMeter
+    )
+    | union (
+        database('Ingestion').Prices_final_v1_4
+        // Convert real to decimal
+        | extend
+            ContractedUnitPrice                  = todecimal(ContractedUnitPrice),
+            ListUnitPrice                        = todecimal(ListUnitPrice),
+            x_BaseUnitPrice                      = todecimal(x_BaseUnitPrice),
+            x_CommitmentDiscountNormalizedRatio  = todecimal(x_CommitmentDiscountNormalizedRatio),
+            x_ContractedUnitPriceDiscount        = todecimal(x_ContractedUnitPriceDiscount),
+            x_ContractedUnitPriceDiscountPercent = todecimal(x_ContractedUnitPriceDiscountPercent),
+            x_EffectiveUnitPrice                 = todecimal(x_EffectiveUnitPrice),
+            x_EffectiveUnitPriceDiscount         = todecimal(x_EffectiveUnitPriceDiscount),
+            x_EffectiveUnitPriceDiscountPercent  = todecimal(x_EffectiveUnitPriceDiscountPercent),
+            x_PricingBlockSize                   = todecimal(x_PricingBlockSize),
+            x_SkuIncludedQuantity                = todecimal(x_SkuIncludedQuantity),
+            x_SkuTier                            = todecimal(x_SkuTier),
+            x_TotalUnitPriceDiscount             = todecimal(x_TotalUnitPriceDiscount),
+            x_TotalUnitPriceDiscountPercent      = todecimal(x_TotalUnitPriceDiscountPercent)
         // Rename columns
         | project-rename
             x_PricingCurrency = PricingCurrency,
@@ -315,6 +407,14 @@ Recommendations_v1_0()
             x_EffectiveCostBefore  = todecimal(x_EffectiveCostBefore),
             x_EffectiveCostSavings = todecimal(x_EffectiveCostSavings)
     )
+    | union (
+        database('Ingestion').Recommendations_final_v1_4
+        // Convert real to decimal
+        | extend
+            x_EffectiveCostAfter   = todecimal(x_EffectiveCostAfter),
+            x_EffectiveCostBefore  = todecimal(x_EffectiveCostBefore),
+            x_EffectiveCostSavings = todecimal(x_EffectiveCostSavings)
+    )
     | project
         ProviderName,
         SubAccountId,
@@ -349,6 +449,18 @@ Transactions_v1_0()
         | project-rename
             x_InvoiceId = InvoiceId
     )
+    | union (
+        database('Ingestion').Transactions_final_v1_4
+        // Convert real to decimal
+        | extend
+            BilledCost           = todecimal(BilledCost),
+            PricingQuantity      = todecimal(PricingQuantity),
+            x_MonetaryCommitment = todecimal(x_MonetaryCommitment),
+            x_Overage            = todecimal(x_Overage)
+        // Rename columns
+        | project-rename
+            x_InvoiceId = InvoiceId
+    )
     | project
         BilledCost,
         BillingAccountId,