diff --git a/.gitignore b/.gitignore index db3a8d4e5..e4fb25448 100644 --- a/.gitignore +++ b/.gitignore @@ -370,3 +370,10 @@ env/ # AI .claude/settings.local.json + +# Auto-generated build artifacts +src/templates/finops-hub-copilot-studio/knowledge/query-catalog.md +.gate/ +todo/ +done/ +release/scloud-occurrence-report.md diff --git a/docs-mslearn/TOC.yml b/docs-mslearn/TOC.yml index 65f398649..4d48e0aff 100644 --- a/docs-mslearn/TOC.yml +++ b/docs-mslearn/TOC.yml @@ -132,14 +132,18 @@ href: toolkit/hubs/private-networking.md - name: Configure scopes href: toolkit/hubs/configure-scopes.md + - name: Configure recommendations + href: toolkit/hubs/configure-recommendations.md - name: Configure dashboards href: toolkit/hubs/configure-dashboards.md - name: Configure AI agents href: toolkit/hubs/configure-ai.md + - name: Configure AI in Copilot Studio + href: toolkit/hubs/configure-ai-copilot-studio.md - name: Configure remote hubs href: toolkit/hubs/configure-remote-hubs.md - name: Savings calculations - href: toolkit/hubs/savings-calculations.md + href: toolkit/hubs/savings-calculations.md - name: Deployment template href: toolkit/hubs/template.md - name: Data model diff --git a/docs-mslearn/best-practices/compute.md b/docs-mslearn/best-practices/compute.md index 21880ff55..403d54a31 100644 --- a/docs-mslearn/best-practices/compute.md +++ b/docs-mslearn/best-practices/compute.md @@ -3,7 +3,7 @@ title: FinOps best practices for compute description: This article provides FinOps best practices for compute services, including cost optimization, efficiency improvements, and insights into Azure resources. author: flanakin ms.author: micflan -ms.date: 04/02/2025 +ms.date: 04/01/2026 ms.topic: concept-article ms.service: finops ms.subservice: finops-learning-resources @@ -11,7 +11,6 @@ ms.reviewer: arclares #customer intent: As a FinOps user, I want to understand what FinOps best practices I should use with compute services. --- - # FinOps best practices for compute This article outlines a collection of proven FinOps practices for compute services. It provides guidance on optimizing costs, improving efficiency, and gaining insights into your compute resources in Azure. The practices are categorized based on the type of compute service, such as virtual machines (VM), Azure Kubernetes Service (AKS), and Azure Functions. @@ -20,9 +19,16 @@ This article outlines a collection of proven FinOps practices for compute servic ## Azure Kubernetes Service -The following section provides an Azure Resource Graph (ARG) query for AKS clusters. The query helps you gain insights into your VMs. +Azure Kubernetes Service (AKS) simplifies deploying and managing containerized applications. It offers serverless Kubernetes, an integrated CI/CD experience, and enterprise-grade security and governance. -### Query - AKS cluster +Related resources: + +- [Azure Kubernetes Service product page](https://azure.microsoft.com/products/kubernetes-service) +- [Azure Kubernetes Service pricing](https://azure.microsoft.com/pricing/details/kubernetes-service) +- [Azure Kubernetes Service documentation](/azure/aks) +- [AKS baseline architecture](/azure/architecture/reference-architectures/containers/aks/baseline-aks) + +### Query: AKS cluster details This ARG query retrieves detailed information about AKS clusters in your Azure environment. @@ -55,6 +61,50 @@ resources AKSname = name ``` +### Use Spot VMs for AKS clusters + +Recommendation: Use Spot VMs for AKS agent pools to reduce compute costs for fault-tolerant, interruptible workloads. + +#### About Spot VMs in AKS + +[Spot VMs](/azure/virtual-machines/spot-vms) take advantage of unused Azure capacity at a significantly reduced cost. When Azure needs the capacity back, the Azure infrastructure evicts Spot VMs. Spot VMs are useful for workloads that can handle interruptions, like batch processing jobs, dev/test environments, and large compute workloads. + +AKS clusters that use autoscaling but don't leverage Spot VMs may be paying more than necessary. By enabling Spot VMs for interruptible workloads, you can significantly reduce compute costs. This recommendation only applies to clusters running workloads that can tolerate interruptions. Not all workloads are suitable for Spot VMs. + + +> [!NOTE] +> [FinOps hubs](../toolkit/hubs/finops-hubs-overview.md) can automatically identify AKS clusters without Spot VMs as an opt-in recommendation. [Learn more](../toolkit/hubs/configure-recommendations.md). + + +#### Query: AKS clusters without Spot VMs + +Use the following ARG query to identify AKS clusters with autoscaling enabled that aren't using Spot VMs. + +**Category** + +Optimization + +**Query** + +```kusto +resources +| where type == 'microsoft.containerservice/managedclusters' +| mvexpand AgentPoolProfiles = properties.agentPoolProfiles +| where AgentPoolProfiles.enableAutoScaling == true + and isnull(AgentPoolProfiles.scaleSetPriority) +| project + ResourceId = id, + AKSName = name, + ProfileName = tostring(AgentPoolProfiles.name), + VMSize = tostring(AgentPoolProfiles.vmSize), + NodeCount = tostring(AgentPoolProfiles.['count']), + MinCount = tostring(AgentPoolProfiles.minCount), + MaxCount = tostring(AgentPoolProfiles.maxCount), + Region = location, + ResourceGroupName = resourceGroup, + SubscriptionId = subscriptionId +``` +
## Virtual machines @@ -82,11 +132,17 @@ Stopped VMs were shut down from within the operating system (for example, using Deallocated VMs are stopped via cloud management APIs in the Azure portal, CLI, PowerShell, or other client tool. When a VM is deallocated, Azure releases the corresponding compute resources. Since compute resources are released, these VMs don't incur compute charges; however, it's important to note that both stopped and deallocated VMs continue to incur charges unrelated to compute, like storage charges from disks. + +> [!NOTE] +> [FinOps hubs](../toolkit/hubs/finops-hubs-overview.md) can automatically identify stopped VMs that aren't deallocated. [Learn more](../toolkit/hubs/configure-recommendations.md). + + #### Identify stopped VMs Use the following Azure Resource Graph (ARG) query to identify stopped VMs that aren't deallocated. It retrieves details about their power state, location, resource group, and subscription ID. + ```kusto resources | where type =~ 'microsoft.compute/virtualmachines' @@ -115,6 +171,7 @@ To learn more about commitment discounts, refer to the [Rate optimization capabi Use the following FinOps hub query to measure overall VM commitment discount coverage. + ```kusto Costs | where ResourceType =~ 'Virtual machine' @@ -187,7 +244,7 @@ Costs To learn more about FinOps hubs, refer to [FinOps hubs](../toolkit/hubs/finops-hubs-overview.md). -### Query - Virtual machine scale set details +### Query: Virtual machine scale set details This query analyzes Virtual Machine Scale Sets in your Azure environment based on their SKU, spot VM priority, and priority mix policy. It provides insights for cost optimization and resource management strategies. @@ -207,7 +264,37 @@ resources | project id, SKU, SpotVMs, SpotPriorityMix, subscriptionId, resourceGroup, location ``` -### Query - Virtual machine processor type analysis +### Migrate to managed disks + +Recommendation: Migrate VMs using unmanaged disks to managed disks to improve reliability, simplify management, and prepare for the retirement of unmanaged disks. + +#### About unmanaged disks + +[Unmanaged disks](/azure/virtual-machines/unmanaged-disks-deprecation) store VHD files as page blobs in Azure Storage accounts, requiring you to manage storage account capacity, performance, and security yourself. Managed disks simplify disk management by handling storage account management for you, providing better reliability with availability sets, more granular access control, and support for newer features like disk encryption and bursting. Microsoft has announced the retirement of unmanaged disks, so migrating to managed disks is both a cost optimization and a compliance step. + + +> [!NOTE] +> [FinOps hubs](../toolkit/hubs/finops-hubs-overview.md) can automatically identify VMs using unmanaged disks. [Learn more](../toolkit/hubs/configure-recommendations.md). + + +#### Identify VMs with unmanaged disks + +Use the following ARG query to identify VMs that are still using unmanaged disks. + +```kusto +resources +| where type =~ 'microsoft.compute/virtualmachines' +| where isnull(properties.storageProfile.osDisk.managedDisk) +| project + ResourceId = tolower(id), + ResourceName = name, + OsDiskVhd = tostring(properties.storageProfile.osDisk.vhd.uri), + Region = location, + ResourceGroupName = resourceGroup, + SubscriptionId = subscriptionId +``` + +### Query: Virtual machine processor type analysis This query identifies the processor type (ARM, AMD, or Intel) used by VMs in your Azure environment. It helps in understanding the distribution of VMs across different processor architectures, which is useful for optimizing workload performance and cost efficiency. @@ -249,19 +336,145 @@ resources | project vmName = name, processorType, vmSize, resourceGroup ``` +### Use Azure Hybrid Benefit for Windows VMs + +Recommendation: Enable Azure Hybrid Benefit for Windows VMs to reduce licensing costs by using existing on-premises Windows Server licenses. + +#### About Azure Hybrid Benefit for Windows + +[Azure Hybrid Benefit](/azure/virtual-machines/windows/hybrid-use-benefit-licensing) lets you use your on-premises Windows Server licenses with Software Assurance or Windows Server subscription to run Windows VMs in Azure at a reduced cost. Instead of paying for a full Windows Server license with each VM, you can bring your existing licenses and only pay for the base compute cost. This recommendation only applies if your organization has qualifying on-premises Windows Server licenses. + + +> [!NOTE] +> [FinOps hubs](../toolkit/hubs/finops-hubs-overview.md) can automatically identify Windows VMs without Azure Hybrid Benefit as an opt-in recommendation. [Learn more](../toolkit/hubs/configure-recommendations.md). + + +#### Query: Windows VMs without Azure Hybrid Benefit + +Use the following ARG query to identify Windows VMs and scale sets that aren't leveraging Azure Hybrid Benefit. The query excludes dev/test subscriptions, which already have discounted licensing. + +**Category** + +Optimization + +**Query** + + + +```kusto +resourcecontainers +| where type =~ 'Microsoft.Resources/subscriptions' +| where tostring(properties.subscriptionPolicies.quotaId) !has 'MSDNDevTest_2014-09-01' +| project SubscriptionName = name, subscriptionId +| join ( + resources + | where type =~ 'microsoft.compute/virtualmachines' + or type =~ 'microsoft.compute/virtualMachineScaleSets' + | where tostring(properties.storageProfile.osDisk.osType) == 'Windows' + or tostring(properties.virtualMachineProfile.storageProfile.osDisk.osType) == 'Windows' + | where tostring(properties.['licenseType']) !has 'Windows' + and tostring(properties.virtualMachineProfile.['licenseType']) != 'Windows_Server' + | project + ResourceId = id, + ResourceName = name, + VMSize = tostring(properties.hardwareProfile.vmSize), + LicenseType = tostring(properties.['licenseType']), + Region = location, + ResourceGroupName = resourceGroup, + subscriptionId +) on subscriptionId +| project + ResourceId, + ResourceName, + VMSize, + LicenseType, + Region, + ResourceGroupName, + SubscriptionName, + SubscriptionId = subscriptionId +``` + +### Use Azure Hybrid Benefit for SQL VMs + +Recommendation: Enable Azure Hybrid Benefit for SQL Server VMs to reduce licensing costs by using existing on-premises SQL Server licenses. + +#### About Azure Hybrid Benefit for SQL VMs + +[Azure Hybrid Benefit for SQL Server](/azure/azure-sql/virtual-machines/windows/licensing-model-azure-hybrid-benefit-ahb-change) lets you use your on-premises SQL Server licenses with Software Assurance to run SQL Server VMs in Azure at a reduced cost. This benefit applies to Standard and Enterprise editions (Developer and Express editions are already free and don't need Azure Hybrid Benefit). This recommendation only applies if your organization has qualifying on-premises SQL Server licenses with Software Assurance. + + +> [!NOTE] +> [FinOps hubs](../toolkit/hubs/finops-hubs-overview.md) can automatically identify SQL VMs without Azure Hybrid Benefit as an opt-in recommendation. [Learn more](../toolkit/hubs/configure-recommendations.md). + + +#### Query: SQL VMs without Azure Hybrid Benefit + +Use the following ARG query to identify SQL Server VMs that aren't leveraging Azure Hybrid Benefit. The query excludes dev/test subscriptions and Developer/Express editions. + +**Category** + +Optimization + +**Query** + +```kusto +resourcecontainers +| where type =~ 'Microsoft.Resources/subscriptions' +| where tostring(properties.subscriptionPolicies.quotaId) !has 'MSDNDevTest_2014-09-01' +| project SubscriptionName = name, subscriptionId +| join ( + resources + | where type =~ 'Microsoft.SqlVirtualMachine/SqlVirtualMachines' + and tostring(properties.['sqlServerLicenseType']) != 'AHUB' + | project + ResourceId = id, + ResourceName = name, + LicenseType = tostring(properties.['sqlServerLicenseType']), + SQLVersion = tostring(properties.['sqlImageOffer']), + SQLSKU = tostring(properties.['sqlImageSku']), + Region = location, + ResourceGroupName = resourceGroup, + subscriptionId +) on subscriptionId +| join ( + resources + | where type =~ 'Microsoft.Compute/virtualMachines' + | project + ResourceName = tolower(name), + VMSize = tostring(properties.hardwareProfile.vmSize), + subscriptionId +) on ResourceName +| where SQLSKU != 'Developer' and SQLSKU != 'Express' +| project + ResourceId, + ResourceName, + VMSize, + LicenseType, + SQLVersion, + SQLSKU, + Region, + ResourceGroupName, + SubscriptionName, + SubscriptionId = subscriptionId +``` +
## Give feedback Let us know how we're doing with a quick review. We use these reviews to improve and expand FinOps tools and resources. + > [!div class="nextstepaction"] > [Give feedback](https://portal.azure.com/#view/HubsExtension/InProductFeedbackBlade/extensionName/FinOpsToolkit/cesQuestion/How%20easy%20or%20hard%20is%20it%20to%20use%20FinOps%20toolkit%20tools%20and%20resources%3F/cvaQuestion/How%20valuable%20is%20the%20FinOps%20toolkit%3F/surveyId/FTK/bladeName/Guide.BestPractices/featureName/Compute) + If you're looking for something specific, vote for an existing or create a new idea. Share ideas with others to get more votes. We focus on ideas with the most votes. + > [!div class="nextstepaction"] > [Vote on or suggest ideas](https://github.com/microsoft/finops-toolkit/issues?q=is%3Aissue+is%3Aopen+sort%3Areactions-%252B1-desc) +
diff --git a/docs-mslearn/best-practices/databases.md b/docs-mslearn/best-practices/databases.md index f817eb54d..60856ffb2 100644 --- a/docs-mslearn/best-practices/databases.md +++ b/docs-mslearn/best-practices/databases.md @@ -3,7 +3,7 @@ title: FinOps best practices for Databases description: This article outlines a collection of proven FinOps practices for database services. author: flanakin ms.author: micflan -ms.date: 04/02/2025 +ms.date: 04/01/2026 ms.topic: concept-article ms.service: finops ms.subservice: finops-learning-resources @@ -11,7 +11,6 @@ ms.reviewer: arclares #customer intent: As a FinOps user, I want to understand what FinOps best practices I should use with database services. --- - # FinOps best practices for Databases This article outlines a collection of proven FinOps practices for database services. It provides strategies for optimizing costs, improving efficiency, and using Azure Resource Graph (ARG) queries to gain insights into your database resources. By following these practices, you can ensure that your database services are cost-effective and aligned with your organization's financial goals. @@ -42,7 +41,7 @@ advisorresources | where properties.impactedField == 'microsoft.documentdb/databaseaccounts' and properties.recommendationTypeId == '8b993855-1b3f-4392-8860-6ed4f5afd8a7' | order by id asc -| project +| project id, subscriptionId, resourceGroup, CosmosDBAccountName = properties.extendedProperties.GlobalDatabaseAccountName, DatabaseName = properties.extendedProperties.DatabaseName, @@ -79,7 +78,7 @@ advisorresources ' 6aa7a0df-192f-4dfa-bd61-f43db4843e7d' ) | order by id asc -| project +| project id, subscriptionId, resourceGroup, CosmosDBAccountName = properties.extendedProperties.GlobalDatabaseAccountName, DatabaseName = properties.extendedProperties.DatabaseName, @@ -117,7 +116,14 @@ resources ## SQL Databases -The following sections provide ARG queries for SQL Databases. These queries help you identify SQL databases that might be idle, old, in development, or used for testing purposes. By analyzing these databases, you can optimize costs and improve efficiency by decommissioning or repurposing underutilized resources. +Azure SQL Database is a fully managed platform as a service (PaaS) database engine that handles most database management functions such as upgrading, patching, backups, and monitoring without user involvement. Elastic Pools allow you to share resources among multiple databases to optimize cost. + +Related resources: + +- [Azure SQL Database product page](https://azure.microsoft.com/products/azure-sql/database) +- [Azure SQL Database pricing](https://azure.microsoft.com/pricing/details/azure-sql-database/single) +- [Azure SQL Database documentation](/azure/azure-sql/database) +- [SQL Database performance guidance](/azure/architecture/checklist/data-ops) ### Query: SQL DB idle @@ -139,15 +145,22 @@ resources | project id, SQLDBName, Type, Tier, resourceGroup, Location, subscriptionId ``` -### Query: Unused Elastic Pools analysis +### Remove unused Elastic Pools -This ARG query identifies potentially idle Elastic Pools in your Azure SQL environment by analyzing the number of databases associated with each Elastic Pool. +Recommendation: Remove Elastic Pools that have no associated databases to avoid unnecessary costs. -**Category** +#### About unused Elastic Pools -Optimization +SQL Elastic Pools let multiple databases share a common pool of resources. When an Elastic Pool has no databases, it still incurs charges based on its configured eDTUs or vCores. Removing empty Elastic Pools eliminates these unnecessary costs. -**Query** + +> [!NOTE] +> [FinOps hubs](../toolkit/hubs/finops-hubs-overview.md) can automatically identify unused Elastic Pools. [Learn more](../toolkit/hubs/configure-recommendations.md). + + +#### Identify unused Elastic Pools + +Use the following ARG query to identify Elastic Pools with no associated databases. ```kusto resources @@ -163,7 +176,7 @@ resources | where type == "microsoft.sql/servers/databases" | extend elasticPoolId = tolower(tostring(properties.elasticPoolId)) ) on elasticPoolId -| summarize databaseCount = countif(isnotempty(elasticPoolId1)) by +| summarize databaseCount = countif(isnotempty(elasticPoolId1)) by elasticPoolId, elasticPoolName, serverResourceGroup = resourceGroup, @@ -188,13 +201,17 @@ resources Let us know how we're doing with a quick review. We use these reviews to improve and expand FinOps tools and resources. + > [!div class="nextstepaction"] > [Give feedback](https://portal.azure.com/#view/HubsExtension/InProductFeedbackBlade/extensionName/FinOpsToolkit/cesQuestion/How%20easy%20or%20hard%20is%20it%20to%20use%20FinOps%20toolkit%20tools%20and%20resources%3F/cvaQuestion/How%20valuable%20is%20the%20FinOps%20toolkit%3F/surveyId/FTK/bladeName/Guide.BestPractices/featureName/Databases) + If you're looking for something specific, vote for an existing or create a new idea. Share ideas with others to get more votes. We focus on ideas with the most votes. + > [!div class="nextstepaction"] > [Vote on or suggest ideas](https://github.com/microsoft/finops-toolkit/issues?q=is%3Aissue+is%3Aopen+sort%3Areactions-%252B1-desc) +
diff --git a/docs-mslearn/best-practices/general.md b/docs-mslearn/best-practices/general.md index 6f763d01c..6aef61346 100644 --- a/docs-mslearn/best-practices/general.md +++ b/docs-mslearn/best-practices/general.md @@ -3,7 +3,7 @@ title: FinOps best practices for general resource management description: This article outlines proven FinOps practices for Microsoft Cloud services, focusing on cost optimization, efficiency improvements, and resource insights. author: flanakin ms.author: micflan -ms.date: 04/02/2025 +ms.date: 04/01/2026 ms.topic: concept-article ms.service: finops ms.subservice: finops-learning-resources @@ -11,13 +11,59 @@ ms.reviewer: arclares #customer intent: As a FinOps user, I want to understand what FinOps best practices I should use with Microsoft Cloud services. --- - # FinOps best practices for general resource management This article outlines a collection of general FinOps best practices that can be applied to various Microsoft Cloud services. It includes strategies for optimizing costs, improving efficiency, and using Azure Resource Graph (ARG) queries to gain insights into your resources. By following these practices, you can ensure that your cloud services are cost-effective and aligned with your organization's financial goals.
+## Azure Advisor + +Azure Advisor is a personalized cloud consultant that helps you follow best practices to optimize your Azure deployments. It analyzes your resource configuration and usage telemetry and recommends solutions that can help you reduce costs, improve performance, increase reliability, and enhance security. + +Related resources: + +- [Azure Advisor product page](https://azure.microsoft.com/products/advisor) +- [Azure Advisor documentation](/azure/advisor) +- [Azure Advisor cost recommendations](/azure/advisor/advisor-cost-recommendations) + +### Review Azure Advisor cost recommendations + +Recommendation: Regularly review and act on Azure Advisor cost recommendations to identify savings opportunities across your environment. + +#### About Azure Advisor cost recommendations + +Azure Advisor analyzes your resource configuration and usage to identify cost-saving opportunities. Cost recommendations include actions like resizing or shutting down underutilized resources, purchasing reservations, and right-sizing workloads. By reviewing these recommendations regularly, you can ensure your environment stays optimized as usage patterns change. + + +> [!NOTE] +> [FinOps hubs](../toolkit/hubs/finops-hubs-overview.md) can automatically ingest Azure Advisor cost recommendations. [Learn more](../toolkit/hubs/configure-recommendations.md). + + +#### Identify Azure Advisor cost recommendations + +Use the following ARG query to surface all cost recommendations from Azure Advisor, including impact details and extended properties. + +```kusto +advisorresources +| where type == 'microsoft.advisor/recommendations' +| where properties.category == 'Cost' +| project + id, + subscriptionId, + resourceGroup, + ResourceId = tostring(properties.resourceMetadata.resourceId), + ResourceType = tostring(properties.impactedField), + Impact = tostring(properties.impact), + Description = tostring(properties.shortDescription.problem), + Solution = tostring(properties.shortDescription.solution), + RecommendationTypeId = tostring(properties.recommendationTypeId), + LastUpdated = tostring(properties.lastUpdated), + ExtendedProperties = properties.extendedProperties +``` + +
+ ## Carbon Optimization The following section provides an ARG query for carbon optimization. It helps you gain insights into your Azure resources and identify opportunities to reduce carbon emissions. By analyzing recommendations from Azure Advisor, you can optimize your cloud infrastructure for sustainability and environmental impact. @@ -61,13 +107,17 @@ advisorresources Let us know how we're doing with a quick review. We use these reviews to improve and expand FinOps tools and resources. + > [!div class="nextstepaction"] > [Give feedback](https://portal.azure.com/#view/HubsExtension/InProductFeedbackBlade/extensionName/FinOpsToolkit/cesQuestion/How%20easy%20or%20hard%20is%20it%20to%20use%20FinOps%20toolkit%20tools%20and%20resources%3F/cvaQuestion/How%20valuable%20is%20the%20FinOps%20toolkit%3F/surveyId/FTK/bladeName/Guide.BestPractices/featureName/General) + If you're looking for something specific, vote for an existing or create a new idea. Share ideas with others to get more votes. We focus on ideas with the most votes. + > [!div class="nextstepaction"] > [Vote on or suggest ideas](https://github.com/microsoft/finops-toolkit/issues?q=is%3Aissue+is%3Aopen+sort%3Areactions-%252B1-desc) +
diff --git a/docs-mslearn/best-practices/library.md b/docs-mslearn/best-practices/library.md index a7c8933c2..4fb00227f 100644 --- a/docs-mslearn/best-practices/library.md +++ b/docs-mslearn/best-practices/library.md @@ -3,7 +3,7 @@ title: FinOps best practices library description: Discover essential FinOps best practices to optimize cost efficiency and governance for your Azure resources. author: flanakin ms.author: micflan -ms.date: 02/24/2026 +ms.date: 04/01/2026 ms.topic: concept-article ms.service: finops ms.subservice: finops-learning-resources diff --git a/docs-mslearn/best-practices/networking.md b/docs-mslearn/best-practices/networking.md index 074e67b2b..dc5796416 100644 --- a/docs-mslearn/best-practices/networking.md +++ b/docs-mslearn/best-practices/networking.md @@ -3,7 +3,7 @@ title: FinOps best practices for Networking description: This article outlines proven FinOps practices for networking services, focusing on cost optimization, efficiency improvements, and resource insights. author: flanakin ms.author: micflan -ms.date: 04/02/2025 +ms.date: 04/01/2026 ms.topic: concept-article ms.service: finops ms.subservice: finops-learning-resources @@ -11,8 +11,9 @@ ms.reviewer: arclares #customer intent: As a FinOps user, I want to understand what FinOps best practices I should use with networking services. --- - + # FinOps best practices for Networking + This article outlines proven FinOps practices for networking services. They focus on cost optimization, efficiency improvements, and resource insights. @@ -33,7 +34,7 @@ Optimization **Query** ```kusto -resources +resources | where type =~ 'Microsoft.Network/azureFirewalls' and properties.sku.tier=="Premium" | project FWID=id, firewallName=name, SkuTier=tostring(properties.sku.tier), resourceGroup, location | join kind=inner ( @@ -97,17 +98,30 @@ resources ## Application Gateway -The following section provides an ARG queries for Azure Application Gateway. It helps you gain insights into your Azure Application Gateway resources and ensure they're configured with the appropriate settings. +Azure Application Gateway is a web traffic load balancer that enables you to manage traffic to your web applications. It provides application-level routing and load balancing services that let you build a scalable and highly available web front end in Azure. -### Query: Idle application gateways +Related resources: -This ARG query analyzes application gateways and their associated backend pools within your Azure environment. It provides insights into which application gateways have empty backend pools, indicating they might be idle and potentially unnecessary. +- [Application Gateway product page](https://azure.microsoft.com/products/application-gateway) +- [Application Gateway pricing](https://azure.microsoft.com/pricing/details/application-gateway) +- [Application Gateway documentation](/azure/application-gateway) -**Category** +### Remove idle application gateways -Optimization +Recommendation: Remove application gateways that don't have any backend pools to avoid unnecessary costs. -**Query** +#### About idle application gateways + +Application gateways without any backend pool targets aren't actively routing traffic and may represent unused resources. These idle gateways continue to incur costs even though they serve no function. + + +> [!NOTE] +> [FinOps hubs](../toolkit/hubs/finops-hubs-overview.md) can automatically identify idle application gateways. [Learn more](../toolkit/hubs/configure-recommendations.md). + + +#### Identify idle application gateways + +Use the following ARG query to identify application gateways with empty backend pools. ```kusto resources @@ -138,21 +152,105 @@ resources | order by id asc ``` +### Upgrade classic application gateways + +Recommendation: Upgrade Application Gateway v1 SKU to v2 before the v1 retirement date to maintain support and access improved features. + +#### About classic application gateways + +Application Gateway v1 SKU (Standard and WAF) is being retired. The v2 SKU offers autoscaling, zone redundancy, and improved performance. Migrating to v2 ensures continued support and may reduce costs through autoscaling, which automatically adjusts the number of instances based on traffic. + + +> [!NOTE] +> [FinOps hubs](../toolkit/hubs/finops-hubs-overview.md) can automatically identify classic application gateways using v1 SKU. [Learn more](../toolkit/hubs/configure-recommendations.md). + + +#### Identify classic application gateways + +Use the following ARG query to identify application gateways still using the v1 SKU. + +```kusto +resources +| where type =~ 'microsoft.network/applicationgateways' +| where properties.sku.tier in ('Standard', 'WAF') +| project + ResourceId = tolower(id), + ResourceName = name, + SKUTier = tostring(properties.sku.tier), + Region = location, + ResourceGroupName = resourceGroup, + SubscriptionId = subscriptionId +``` + +
+ +## DDoS Protection + +Azure DDoS Protection provides countermeasures against the most sophisticated DDoS threats. It provides enhanced DDoS mitigation capabilities for your application and resources deployed in your virtual networks. + +Related resources: + +- [Azure DDoS Protection product page](https://azure.microsoft.com/products/ddos-protection) +- [Azure DDoS Protection pricing](https://azure.microsoft.com/pricing/details/ddos-protection) +- [Azure DDoS Protection documentation](/azure/ddos-protection) + +### Remove unassociated DDoS protection plans + +Recommendation: Remove DDoS protection plans that aren't associated with any virtual network to avoid unnecessary costs. + +#### About unassociated DDoS protection plans + +DDoS protection plans incur a fixed monthly charge. Plans that aren't associated with any virtual network provide no protection but still generate costs. Removing unused plans eliminates unnecessary spending. + + +> [!NOTE] +> [FinOps hubs](../toolkit/hubs/finops-hubs-overview.md) can automatically identify unassociated DDoS protection plans. [Learn more](../toolkit/hubs/configure-recommendations.md). + + +#### Identify unassociated DDoS protection plans + +Use the following ARG query to identify DDoS protection plans that aren't associated with any virtual network. + +```kusto +resources +| where type =~ 'microsoft.network/ddosprotectionplans' +| where isnull(properties.virtualNetworks) or array_length(properties.virtualNetworks) == 0 +| project + ResourceId = tolower(id), + ResourceName = name, + Region = location, + ResourceGroupName = resourceGroup, + SubscriptionId = subscriptionId +``` +
## ExpressRoute -The following section provides an ARG query for ExpressRoute. It helps you gain insights into your ExpressRoute circuits and ensure they're configured with the appropriate settings. +Azure ExpressRoute lets you extend your on-premises networks into the Microsoft cloud over a private connection. ExpressRoute circuits incur monthly charges based on the SKU and bandwidth provisioned. -### Query: Idle ExpressRoute circuits +Related resources: -This ARG query analyzes ExpressRoute circuits within your Azure environment to identify any without a completed circuit. +- [ExpressRoute product page](https://azure.microsoft.com/products/expressroute) +- [ExpressRoute pricing](https://azure.microsoft.com/pricing/details/expressroute) +- [ExpressRoute documentation](/azure/expressroute) -**Category** +### Remove unprovisioned ExpressRoute circuits -Optimization +Recommendation: Delete or provision ExpressRoute circuits that are in a not-provisioned state to avoid unnecessary charges. -**Query** +#### About unprovisioned ExpressRoute circuits + +ExpressRoute circuits that remain in a "NotProvisioned" state aren't actively carrying traffic but still incur monthly charges. These circuits may have been created but never completed with the service provider. Identifying and removing them eliminates unnecessary costs. + + +> [!NOTE] +> [FinOps hubs](../toolkit/hubs/finops-hubs-overview.md) can automatically identify unprovisioned ExpressRoute circuits. [Learn more](../toolkit/hubs/configure-recommendations.md). + + +#### Identify unprovisioned ExpressRoute circuits + +Use the following ARG query to identify ExpressRoute circuits in a not-provisioned state. ```kusto resources @@ -179,17 +277,30 @@ resources ## Load Balancer -The following section provides an ARG query for Azure Load Balancer. It helps you gain insights into your Azure load balancer resources and ensure they're configured with the appropriate settings. +Azure Load Balancer operates at layer 4 of the OSI model and distributes inbound traffic across healthy backend pool instances. It provides high availability by monitoring the health of backend instances and automatically rerouting traffic away from unhealthy ones. -### Query: Idle load balancers +Related resources: -This ARG query analyzes Azure load balancers and their associated backend pools within your Azure environment. It provides insights into which load balancers have empty backend pools, indicating they might be idle and potentially unnecessary. +- [Load Balancer product page](https://azure.microsoft.com/products/load-balancer) +- [Load Balancer pricing](https://azure.microsoft.com/pricing/details/load-balancer) +- [Load Balancer documentation](/azure/load-balancer) -**Category** +### Remove idle load balancers -Optimization +Recommendation: Remove load balancers that don't have any backend pools to avoid unnecessary costs. -**Query** +#### About idle load balancers + +Load balancers without backend pool targets aren't actively distributing traffic and may represent unused resources. Standard SKU load balancers incur costs even when idle, so removing unused instances can reduce unnecessary spending. + + +> [!NOTE] +> [FinOps hubs](../toolkit/hubs/finops-hubs-overview.md) can automatically identify idle load balancers. [Learn more](../toolkit/hubs/configure-recommendations.md). + + +#### Identify idle load balancers + +Use the following ARG query to identify Standard SKU load balancers with empty backend pools. ```kusto resources @@ -212,6 +323,151 @@ resources subscriptionId ``` +### Upgrade Basic load balancers + +Recommendation: Upgrade load balancers using the retired Basic SKU to Standard for better performance, security, and continued support. + +#### About Basic load balancers + +The Basic SKU for Azure Load Balancer was retired on September 30, 2025. Basic load balancers don't provide an SLA, lack availability zone support, and have limited diagnostic capabilities. Upgrading to Standard SKU provides improved reliability, performance, and security features. + + +> [!NOTE] +> [FinOps hubs](../toolkit/hubs/finops-hubs-overview.md) can automatically identify Basic load balancers. [Learn more](../toolkit/hubs/configure-recommendations.md). + + +#### Identify Basic load balancers + +Use the following ARG query to identify load balancers using the Basic SKU. + +```kusto +resources +| where type =~ 'microsoft.network/loadbalancers' +| where sku.name =~ 'Basic' +| project + ResourceId = tolower(id), + ResourceName = name, + SKUName = tostring(sku.name), + Region = location, + ResourceGroupName = resourceGroup, + SubscriptionId = subscriptionId +``` + +
+ +## NAT Gateway + +Azure NAT Gateway provides outbound internet connectivity for virtual networks. NAT gateways simplify outbound-only internet connectivity by providing a managed, highly available SNAT service. + +Related resources: + +- [NAT Gateway product page](https://azure.microsoft.com/products/azure-nat-gateway) +- [NAT Gateway pricing](https://azure.microsoft.com/pricing/details/azure-nat-gateway) +- [NAT Gateway documentation](/azure/nat-gateway) + +### Remove orphaned NAT gateways + +Recommendation: Remove NAT gateways that aren't associated with any subnet to avoid unnecessary charges. + +#### About orphaned NAT gateways + +NAT gateways incur hourly charges and data processing costs. Gateways that aren't associated with any subnet aren't providing outbound connectivity and represent wasted spend. These orphaned gateways may remain after a subnet or virtual network is reconfigured. + + +> [!NOTE] +> [FinOps hubs](../toolkit/hubs/finops-hubs-overview.md) can automatically identify orphaned NAT gateways. [Learn more](../toolkit/hubs/configure-recommendations.md). + + +#### Identify orphaned NAT gateways + +Use the following ARG query to identify NAT gateways not associated with any subnet. + +```kusto +resources +| where type == "microsoft.network/natgateways" +| where isnull(properties.subnets) or array_length(properties.subnets) == 0 +| project + id, + GWName = name, + SKUName = tostring(sku.name), + SKUTier = tostring(sku.tier), + Location = location, + resourceGroup = tostring(strcat('/subscriptions/', subscriptionId, '/resourceGroups/', resourceGroup)), + subnets = properties.subnets, + subscriptionId +``` + +
+ +## Network Interface + +Azure network interfaces (NICs) enable Azure VMs to communicate with internet, Azure, and on-premises resources. NICs don't incur direct charges, but orphaned NICs can indicate missed cleanup opportunities and complicate resource management. + +### Remove unattached network interfaces + +Recommendation: Remove network interfaces that aren't attached to any virtual machine or private endpoint to keep your environment clean and reduce management overhead. + +#### About unattached network interfaces + +When a VM is deleted, its associated network interfaces may not be cleaned up automatically. These orphaned NICs can accumulate over time, cluttering your environment and potentially retaining associated resources like public IPs. While NICs don't incur direct charges, cleaning them up simplifies resource management and may reveal other orphaned resources. + + +> [!NOTE] +> [FinOps hubs](../toolkit/hubs/finops-hubs-overview.md) can automatically identify unattached network interfaces. [Learn more](../toolkit/hubs/configure-recommendations.md). + + +#### Identify unattached network interfaces + +Use the following ARG query to identify network interfaces not attached to any VM or private endpoint. + +```kusto +resources +| where type =~ 'microsoft.network/networkinterfaces' +| where isnull(properties.virtualMachine) and isnull(properties.privateEndpoint) +| project + ResourceId = tolower(id), + ResourceName = name, + PrivateIP = tostring(properties.ipConfigurations[0].properties.privateIPAddress), + Region = location, + ResourceGroupName = resourceGroup, + SubscriptionId = subscriptionId +``` + +
+ +## Network Security Group + +Network security groups (NSGs) filter network traffic to and from Azure resources in a virtual network. NSGs contain security rules that allow or deny inbound and outbound network traffic. + +### Remove empty network security groups + +Recommendation: Remove network security groups that aren't associated with any network interface or subnet to simplify your environment and reduce management overhead. + +#### About empty network security groups + +NSGs that aren't associated with any network interface or subnet aren't actively filtering traffic. These unused resources can accumulate during infrastructure changes, adding clutter and complicating security audits. Removing them simplifies network management and helps maintain a clean environment. + + +> [!NOTE] +> [FinOps hubs](../toolkit/hubs/finops-hubs-overview.md) can automatically identify empty network security groups. [Learn more](../toolkit/hubs/configure-recommendations.md). + + +#### Identify empty network security groups + +Use the following ARG query to identify NSGs not associated with any network interface or subnet. + +```kusto +resources +| where type =~ 'microsoft.network/networksecuritygroups' +| where isnull(properties.networkInterfaces) and isnull(properties.subnets) +| project + ResourceId = tolower(id), + ResourceName = name, + Region = location, + ResourceGroupName = resourceGroup, + SubscriptionId = subscriptionId +``` +
## Private DNS @@ -243,17 +499,60 @@ resources ## Public IP address -The following sections provide ARG queries for public IP addresses. They help you gain insights into your public IP address resources and ensure they're configured with the appropriate settings. +Azure public IP addresses enable Azure resources to communicate with the internet and other public-facing Azure services. Public IP addresses are assigned to resources such as virtual machines, load balancers, and application gateways. Static public IP addresses incur costs whether or not they're associated with a resource. -### Query: Idle public IP addresses +Related resources: -This ARG query analyzes Azure public IP addresses. It provides insights into which public IPs are idle and potentially unnecessary. +- [Public IP addresses pricing](https://azure.microsoft.com/pricing/details/ip-addresses) +- [Public IP addresses documentation](/azure/virtual-network/ip-services/public-ip-addresses) -**Category** +### Upgrade Basic public IPs -Optimization +Recommendation: Upgrade public IP addresses using the retired Basic SKU to Standard for better security and continued support. -**Query** +#### About Basic public IPs + +The Basic SKU for Azure public IP addresses was retired on September 30, 2025. Basic public IPs lack zone redundancy, don't support routing preference, and are open to inbound traffic by default. Upgrading to Standard SKU provides zone redundancy, secure-by-default behavior (closed to inbound traffic), and support for routing preferences. + + +> [!NOTE] +> [FinOps hubs](../toolkit/hubs/finops-hubs-overview.md) can automatically identify Basic public IPs. [Learn more](../toolkit/hubs/configure-recommendations.md). + + +#### Identify Basic public IPs + +Use the following ARG query to identify public IP addresses using the Basic SKU. + +```kusto +resources +| where type =~ 'microsoft.network/publicipaddresses' +| where sku.name =~ 'Basic' +| project + ResourceId = tolower(id), + ResourceName = name, + SKUName = tostring(sku.name), + AllocationMethod = tostring(properties.publicIPAllocationMethod), + Region = location, + ResourceGroupName = resourceGroup, + SubscriptionId = subscriptionId +``` + +### Remove idle public IP addresses + +Recommendation: Remove unattached static public IP addresses to avoid unnecessary networking costs. + +#### About idle public IP addresses + +Static public IP addresses incur costs regardless of whether they're associated with a resource. Unattached public IPs can accumulate over time as resources are deleted but their associated public IPs are left behind. Identifying and removing these orphaned resources can reduce unnecessary costs. + + +> [!NOTE] +> [FinOps hubs](../toolkit/hubs/finops-hubs-overview.md) can automatically identify unattached public IP addresses. [Learn more](../toolkit/hubs/configure-recommendations.md). + + +#### Identify idle public IP addresses + +Use the following ARG query to identify unattached static public IP addresses, including those associated with unattached network interfaces. ```kusto resources @@ -301,7 +600,7 @@ resources ) ``` -### Query: Identify public IP addresses routing method +### Query: Identify public IP addresses routing method This ARG query analyzes public IP addresses and identifies the routing method, allocation method, and SKU. It also analyzes other details of public IP addresses that are associated with an IP configuration. @@ -358,17 +657,29 @@ resources ## Virtual Network Gateway -The following sections provide ARG queries for Virtual Network Gateways. They help you gain insights into your Virtual Network Gateway resources and ensure they're configured with the appropriate settings. +Azure Virtual Network Gateways provide cross-premises connectivity between your Azure virtual networks and on-premises infrastructure. Gateways incur hourly charges based on their SKU. -### Query: Check for idle Virtual Network Gateway +Related resources: -This ARG query analyzes Virtual Network Gateways within your Azure environment to identify any that are idle. +- [VPN Gateway pricing](https://azure.microsoft.com/pricing/details/vpn-gateway) +- [VPN Gateway documentation](/azure/vpn-gateway) -**Category** +### Remove idle VNet gateways -Optimization +Recommendation: Remove virtual network gateways that don't have any active connections to avoid unnecessary charges. -**Query** +#### About idle VNet gateways + +Virtual network gateways incur hourly costs based on their SKU tier, regardless of whether they're actively used. Gateways without any connections aren't providing cross-premises connectivity and represent wasted spend. These idle gateways may remain after a migration or when connectivity requirements change. + + +> [!NOTE] +> [FinOps hubs](../toolkit/hubs/finops-hubs-overview.md) can automatically identify idle VNet gateways. [Learn more](../toolkit/hubs/configure-recommendations.md). + + +#### Identify idle VNet gateways + +Use the following ARG query to identify virtual network gateways without any active connections. ```kusto resources @@ -391,43 +702,23 @@ resources status=id ``` -### Query: Check for idle NAT gateway - -This ARG query analyzes NAT gateways within your Azure environment to identify any that are idle. - -**Category** - -Optimization - -**Query** - -```kusto -resources -| where type == "microsoft.network/natgateways" and isnull(properties.subnets) -| project - id, - GWName = name, - SKUName = tostring(sku.name), - SKUTier = tostring(sku.tier), - Location = location, - resourceGroup = tostring(strcat('/subscriptions/', subscriptionId, '/resourceGroups/', resourceGroup)), - subnet = tostring(properties.subnet), - subscriptionId -``` -
## Give feedback Let us know how we're doing with a quick review. We use these reviews to improve and expand FinOps tools and resources. + > [!div class="nextstepaction"] > [Give feedback](https://portal.azure.com/#view/HubsExtension/InProductFeedbackBlade/extensionName/FinOpsToolkit/cesQuestion/How%20easy%20or%20hard%20is%20it%20to%20use%20FinOps%20toolkit%20tools%20and%20resources%3F/cvaQuestion/How%20valuable%20is%20the%20FinOps%20toolkit%3F/surveyId/FTK/bladeName/Guide.BestPractices/featureName/Networking) + If you're looking for something specific, vote for an existing or create a new idea. Share ideas with others to get more votes. We focus on ideas with the most votes. + > [!div class="nextstepaction"] > [Vote on or suggest ideas](https://github.com/microsoft/finops-toolkit/issues?q=is%3Aissue+is%3Aopen+sort%3Areactions-%252B1-desc) +
diff --git a/docs-mslearn/best-practices/storage.md b/docs-mslearn/best-practices/storage.md index 65dd0d90d..6e61ebd3f 100644 --- a/docs-mslearn/best-practices/storage.md +++ b/docs-mslearn/best-practices/storage.md @@ -3,7 +3,7 @@ title: FinOps best practices for Storage description: This article outlines proven FinOps practices for storage services, focusing on cost optimization, efficiency improvements, and resource insights. author: flanakin ms.author: micflan -ms.date: 04/02/2025 +ms.date: 04/01/2026 ms.topic: concept-article ms.service: finops ms.subservice: finops-learning-resources @@ -11,7 +11,6 @@ ms.reviewer: arclares #customer intent: As a FinOps user, I want to understand what FinOps best practices I should use with storage services. --- - # FinOps best practices for Storage This article outlines a collection of proven FinOps practices for storage services. It provides strategies for optimizing costs, improving efficiency, and using Azure Resource Graph (ARG) queries to gain insights into your storage resources. By following these practices, you can ensure that your storage services are cost-effective and aligned with your organization's financial goals. @@ -73,17 +72,30 @@ resources ## Disks -The following sections provide ARG queries for disk services. These queries help you gain insights into your disk resources and ensure they're configured with the appropriate settings. By analyzing disk snapshots and identifying idle disks, you can optimize your disk services for cost efficiency. +Azure managed disks are block-level storage volumes that are managed by Azure and used with virtual machines. Managed disks provide high availability, scalability, and security for your VM workloads. -### Query: Idle disks +Related resources: -This ARG query identifies idle or unattached managed disks within your Azure environment. +- [Managed disks product page](https://azure.microsoft.com/products/managed-disks) +- [Managed disks pricing](https://azure.microsoft.com/pricing/details/managed-disks) +- [Managed disks documentation](/azure/virtual-machines/managed-disks-overview) -**Category** +### Remove unattached disks -Optimization +Recommendation: Remove or downgrade unattached managed disks to avoid unnecessary storage costs. -**Query** +#### About unattached disks + +When a VM is deleted, its associated managed disks may not be deleted automatically. These unattached (orphaned) disks continue to incur storage costs based on their disk type and size. The query excludes disks that are in active SAS transfer mode or are Azure Site Recovery replica or seed disks, as these are expected to be temporarily unattached. + + +> [!NOTE] +> [FinOps hubs](../toolkit/hubs/finops-hubs-overview.md) can automatically identify unattached disks. [Learn more](../toolkit/hubs/configure-recommendations.md). + + +#### Identify unattached disks + +Use the following ARG query to identify unattached managed disks. ```kusto resources @@ -94,7 +106,7 @@ resources and tags !contains 'ASR-ReplicaDisk' and tags !contains 'asrseeddisk' | extend DiskId=id, DiskIDfull=id, DiskName=name, SKUName=sku.name, SKUTier=sku.tier, DiskSizeGB=tostring(properties.diskSizeGB), Location=location, TimeCreated=tostring(properties.timeCreated), SubId=subscriptionId -| order by DiskId asc +| order by DiskId asc | project DiskId, DiskIDfull, DiskName, DiskSizeGB, SKUName, SKUTier, resourceGroup, Location, TimeCreated, subscriptionId ``` @@ -118,15 +130,22 @@ resources | project id, resourceGroup, location, TimeCreated, subscriptionId ``` -### Query: Snapshot using premium storage +### Downgrade premium snapshots -This ARG query identifies disk snapshots that are utilizing premium storage. +Recommendation: Use Standard storage for managed disk snapshots instead of Premium to reduce storage costs. -**Category** +#### About premium snapshots -Optimization +Managed disk snapshots stored on Premium storage incur higher costs than Standard storage. In most cases, snapshots don't require the performance of Premium storage since they're used for backup and recovery, not active I/O. Downgrading to Standard storage can significantly reduce snapshot costs without affecting their functionality. -**Query** + +> [!NOTE] +> [FinOps hubs](../toolkit/hubs/finops-hubs-overview.md) can automatically identify snapshots using Premium storage. [Learn more](../toolkit/hubs/configure-recommendations.md). + + +#### Identify premium snapshots + +Use the following ARG query to identify managed disk snapshots using Premium storage. ```kusto resources @@ -143,24 +162,36 @@ resources ## Storage accounts -The following section provides an ARG query for storage accounts. It helps you gain insights into your storage resources and ensure they're configured with the appropriate settings. By analyzing storage accounts and identifying legacy storage account types, you can optimize your storage services for cost efficiency. +Azure Storage accounts provide a unique namespace in Azure for your data. Storage accounts have evolved through several generations, and using legacy account kinds may limit access to newer features and optimizations. -### Query: Storage account v1 +Related resources: -This ARG query identifies storage accounts that are still using the legacy v1 kind, which might not provide the same features and efficiencies as newer storage account types. +- [Storage account product page](https://azure.microsoft.com/products/storage) +- [Storage account pricing](https://azure.microsoft.com/pricing/details/storage) +- [Storage account documentation](/azure/storage/common/storage-account-overview) -**Category** +### Upgrade legacy storage accounts -Optimization +Recommendation: Upgrade storage accounts using GPv1 or BlobStorage kind to GPv2 for better pricing tiers, features, and continued support. -**Query** +#### About legacy storage accounts + +Storage accounts using the GPv1 or BlobStorage kind don't support the latest Azure Storage features, such as access tiers for block blobs, lifecycle management policies, and immutability policies. GPv2 storage accounts provide the same features plus additional capabilities at competitive or lower prices. Microsoft recommends upgrading all GPv1 and BlobStorage accounts to GPv2. + + +> [!NOTE] +> [FinOps hubs](../toolkit/hubs/finops-hubs-overview.md) can automatically identify legacy storage accounts. [Learn more](../toolkit/hubs/configure-recommendations.md). + + +#### Identify legacy storage accounts + +Use the following ARG query to identify storage accounts still using GPv1 or BlobStorage kind. ```kusto resources | where type =~ 'Microsoft.Storage/StorageAccounts' and kind !='StorageV2' and kind !='FileStorage' -| where resourceGroup in ({ResourceGroup}) | extend StorageAccountName = name, SAKind = kind, @@ -187,13 +218,17 @@ resources Let us know how we're doing with a quick review. We use these reviews to improve and expand FinOps tools and resources. + > [!div class="nextstepaction"] > [Give feedback](https://portal.azure.com/#view/HubsExtension/InProductFeedbackBlade/extensionName/FinOpsToolkit/cesQuestion/How%20easy%20or%20hard%20is%20it%20to%20use%20FinOps%20toolkit%20tools%20and%20resources%3F/cvaQuestion/How%20valuable%20is%20the%20FinOps%20toolkit%3F/surveyId/FTK/bladeName/Guide.BestPractices/featureName/Storage) + If you're looking for something specific, vote for an existing or create a new idea. Share ideas with others to get more votes. We focus on ideas with the most votes. + > [!div class="nextstepaction"] > [Vote on or suggest ideas](https://github.com/microsoft/finops-toolkit/issues?q=is%3Aissue+is%3Aopen+sort%3Areactions-%252B1-desc) +
diff --git a/docs-mslearn/best-practices/web.md b/docs-mslearn/best-practices/web.md index ba6f4c4c8..d687a57e5 100644 --- a/docs-mslearn/best-practices/web.md +++ b/docs-mslearn/best-practices/web.md @@ -3,7 +3,7 @@ title: FinOps best practices for Web description: This article outlines a collection of proven FinOps practices for web services, focusing on cost optimization, efficiency improvements, and resource insights. author: flanakin ms.author: micflan -ms.date: 02/24/2026 +ms.date: 04/01/2026 ms.topic: concept-article ms.service: finops ms.subservice: finops-learning-resources @@ -46,6 +46,38 @@ resources | order by id asc ``` +### Remove empty App Service plans + +Recommendation: Remove App Service plans that have no apps or functions hosted to avoid unnecessary charges. + +#### About empty App Service plans + +App Service plans define the compute resources for your web apps. Paid plans incur charges based on their configured SKU and instance count, regardless of whether any apps are hosted on them. Empty plans can accumulate during development or when apps are moved between plans. Removing unused plans eliminates unnecessary costs. + + +> [!NOTE] +> [FinOps hubs](../toolkit/hubs/finops-hubs-overview.md) can automatically identify empty App Service plans. [Learn more](../toolkit/hubs/configure-recommendations.md). + + +#### Identify empty App Service plans + +Use the following ARG query to identify App Service plans with no hosted apps. + +```kusto +resources +| where type =~ 'microsoft.web/serverfarms' +| where toint(properties.numberOfSites) == 0 +| where sku.tier !~ 'Free' +| project + ResourceId = tolower(id), + ResourceName = name, + SKUName = tostring(sku.name), + SKUTier = tostring(sku.tier), + Region = location, + ResourceGroupName = resourceGroup, + SubscriptionId = subscriptionId +``` + ### Query: App Service plan details This ARG query retrieves detailed information about Azure App Service Plans within your Azure environment. diff --git a/docs-mslearn/conduct-iteration.md b/docs-mslearn/conduct-iteration.md index 0b87e223a..4ce7c5c3a 100644 --- a/docs-mslearn/conduct-iteration.md +++ b/docs-mslearn/conduct-iteration.md @@ -3,7 +3,7 @@ title: Tutorial - Conduct an iteration description: This tutorial helps you learn how to take an iterative approach to FinOps adoption. author: flanakin ms.author: micflan -ms.date: 02/24/2026 +ms.date: 04/01/2026 ms.topic: tutorial ms.service: finops ms.subservice: finops-learning-resources diff --git a/docs-mslearn/fabric/create-fabric-workspace-finops.md b/docs-mslearn/fabric/create-fabric-workspace-finops.md index 0a13fbe19..b291a58b9 100644 --- a/docs-mslearn/fabric/create-fabric-workspace-finops.md +++ b/docs-mslearn/fabric/create-fabric-workspace-finops.md @@ -3,7 +3,7 @@ title: Create a Fabric workspace for FinOps description: This article guides you through creating and configuring a Microsoft Fabric workspace for FinOps. When completed, you can use Power BI to build reports. author: flanakin ms.author: micflan -ms.date: 02/24/2026 +ms.date: 04/01/2026 ms.topic: how-to ms.service: finops ms.subservice: finops-workspaces diff --git a/docs-mslearn/focus/conformance-full-report.md b/docs-mslearn/focus/conformance-full-report.md index c28344d84..618bdce52 100644 --- a/docs-mslearn/focus/conformance-full-report.md +++ b/docs-mslearn/focus/conformance-full-report.md @@ -3,7 +3,7 @@ title: FOCUS conformance report description: Comprehensive analysis of the Microsoft Cost Management FOCUS dataset's adherence to FOCUS requirements. author: flanakin ms.author: micflan -ms.date: 02/24/2026 +ms.date: 04/01/2026 ms.topic: reference ms.service: finops ms.subservice: finops-learning-resources diff --git a/docs-mslearn/focus/conformance-summary.md b/docs-mslearn/focus/conformance-summary.md index 3019bcee0..70e29a155 100644 --- a/docs-mslearn/focus/conformance-summary.md +++ b/docs-mslearn/focus/conformance-summary.md @@ -3,7 +3,7 @@ title: FOCUS conformance summary description: Summary of FOCUS conformance gaps in the Microsoft Cost Management FOCUS dataset with applicable workarounds. author: flanakin ms.author: micflan -ms.date: 02/24/2026 +ms.date: 04/01/2026 ms.topic: reference ms.service: finops ms.subservice: finops-learning-resources diff --git a/docs-mslearn/focus/convert.md b/docs-mslearn/focus/convert.md index 70d0ef074..678a88ec6 100644 --- a/docs-mslearn/focus/convert.md +++ b/docs-mslearn/focus/convert.md @@ -3,7 +3,7 @@ title: Convert cost and usage data to FOCUS description: This document provides guidance for converting existing Cost Management datasets to the FinOps Open Cost and Usage Specification (FOCUS). author: flanakin ms.author: micflan -ms.date: 02/24/2026 +ms.date: 04/01/2026 ms.topic: concept-article ms.service: finops ms.subservice: finops-learning-resources diff --git a/docs-mslearn/focus/mapping.md b/docs-mslearn/focus/mapping.md index 4f5543044..f95f250b3 100644 --- a/docs-mslearn/focus/mapping.md +++ b/docs-mslearn/focus/mapping.md @@ -3,7 +3,7 @@ title: Update reports to use FOCUS columns description: Learn how to update existing reports from Cost Management actual or amortized datasets to use FOCUS columns. author: flanakin ms.author: micflan -ms.date: 02/24/2026 +ms.date: 04/01/2026 ms.topic: concept-article ms.service: finops ms.subservice: finops-learning-resources diff --git a/docs-mslearn/focus/metadata.md b/docs-mslearn/focus/metadata.md index 59eed5ccf..93160875e 100644 --- a/docs-mslearn/focus/metadata.md +++ b/docs-mslearn/focus/metadata.md @@ -3,7 +3,7 @@ title: FOCUS metadata description: This article provides general information about the FOCUS dataset including the data generator, schema version, and columns included in the dataset. author: flanakin ms.author: micflan -ms.date: 02/24/2026 +ms.date: 04/01/2026 ms.topic: reference ms.service: finops ms.subservice: finops-learning-resources diff --git a/docs-mslearn/focus/validate.md b/docs-mslearn/focus/validate.md index 31687e330..f2563c3e8 100644 --- a/docs-mslearn/focus/validate.md +++ b/docs-mslearn/focus/validate.md @@ -3,7 +3,7 @@ title: Validate FOCUS data description: This article provides reference information that explains how to compare FOCUS data with existing Cost Management datasets. author: flanakin ms.author: micflan -ms.date: 02/24/2026 +ms.date: 04/01/2026 ms.topic: reference ms.service: finops ms.subservice: finops-learning-resources diff --git a/docs-mslearn/focus/what-is-focus.md b/docs-mslearn/focus/what-is-focus.md index 2a706df2f..53114cd5e 100644 --- a/docs-mslearn/focus/what-is-focus.md +++ b/docs-mslearn/focus/what-is-focus.md @@ -3,7 +3,7 @@ title: What is FOCUS? description: Learn about FOCUS, a cloud-agnostic billing data specification that helps optimize cost and usage across cloud, SaaS, and on-premises providers. author: flanakin ms.author: micflan -ms.date: 02/24/2026 +ms.date: 04/01/2026 ms.topic: overview ms.service: finops ms.subservice: finops-learning-resources diff --git a/docs-mslearn/framework/capabilities.md b/docs-mslearn/framework/capabilities.md index e67d50740..6eaf71d2a 100644 --- a/docs-mslearn/framework/capabilities.md +++ b/docs-mslearn/framework/capabilities.md @@ -3,7 +3,7 @@ title: FinOps capabilities description: Learn about what the fundamental building blocks of the FinOps Framework that enable you to maximize business value through the cloud. author: flanakin ms.author: micflan -ms.date: 02/24/2026 +ms.date: 04/01/2026 ms.topic: conceptual ms.service: finops ms.subservice: finops-learning-resources diff --git a/docs-mslearn/framework/finops-framework.md b/docs-mslearn/framework/finops-framework.md index 4a86ac4de..92d2623d9 100644 --- a/docs-mslearn/framework/finops-framework.md +++ b/docs-mslearn/framework/finops-framework.md @@ -3,7 +3,7 @@ title: FinOps Framework overview description: 'Learn about what the FinOps Framework is and how you can use it to accelerate your cost management and optimization goals.' author: flanakin ms.author: micflan -ms.date: 02/24/2026 +ms.date: 04/01/2026 ms.topic: concept-article ms.service: finops ms.subservice: finops-learning-resources diff --git a/docs-mslearn/framework/manage/assessment.md b/docs-mslearn/framework/manage/assessment.md index 0f4b77b81..e02353509 100644 --- a/docs-mslearn/framework/manage/assessment.md +++ b/docs-mslearn/framework/manage/assessment.md @@ -4,7 +4,7 @@ description: This article helps you understand the FinOps assessment capability ms.topic: concept-article author: flanakin ms.author: micflan -ms.date: 02/24/2026 +ms.date: 04/01/2026 ms.service: finops ms.subservice: finops-learning-resources ms.reviewer: micflan diff --git a/docs-mslearn/framework/manage/education.md b/docs-mslearn/framework/manage/education.md index a2445a871..89309a9a2 100644 --- a/docs-mslearn/framework/manage/education.md +++ b/docs-mslearn/framework/manage/education.md @@ -3,7 +3,7 @@ title: FinOps education and enablement description: This article helps you understand the FinOps education and enablement capability within the FinOps Framework and how to implement that in the Microsoft Cloud. author: flanakin ms.author: micflan -ms.date: 02/24/2026 +ms.date: 04/01/2026 ms.topic: concept-article ms.service: finops ms.subservice: finops-learning-resources diff --git a/docs-mslearn/framework/manage/governance.md b/docs-mslearn/framework/manage/governance.md index 6c12166ce..29b188f89 100644 --- a/docs-mslearn/framework/manage/governance.md +++ b/docs-mslearn/framework/manage/governance.md @@ -3,7 +3,7 @@ title: Policy and governance description: This article helps you understand the policy and governance capability within the FinOps Framework and how to implement that in the Microsoft Cloud. author: flanakin ms.author: micflan -ms.date: 02/24/2026 +ms.date: 04/01/2026 ms.topic: concept-article ms.service: finops ms.subservice: finops-learning-resources diff --git a/docs-mslearn/framework/manage/intersecting-disciplines.md b/docs-mslearn/framework/manage/intersecting-disciplines.md index 34e002ce8..127edca3e 100644 --- a/docs-mslearn/framework/manage/intersecting-disciplines.md +++ b/docs-mslearn/framework/manage/intersecting-disciplines.md @@ -3,7 +3,7 @@ title: FinOps and intersecting frameworks description: This article helps you understand the FinOps and intersecting frameworks capability in the FinOps Framework and how to implement that in the Microsoft Cloud. author: flanakin ms.author: micflan -ms.date: 02/24/2026 +ms.date: 04/01/2026 ms.topic: concept-article ms.service: finops ms.subservice: finops-learning-resources diff --git a/docs-mslearn/framework/manage/invoicing-chargeback.md b/docs-mslearn/framework/manage/invoicing-chargeback.md index d22fea279..39dc3b62f 100644 --- a/docs-mslearn/framework/manage/invoicing-chargeback.md +++ b/docs-mslearn/framework/manage/invoicing-chargeback.md @@ -3,7 +3,7 @@ title: Invoicing and chargeback description: This article helps you understand the invoicing and chargeback capability in the FinOps Framework and how to implement that in the Microsoft Cloud. author: flanakin ms.author: micflan -ms.date: 02/24/2026 +ms.date: 04/01/2026 ms.service: finops ms.subservice: finops-learning-resources ms.reviewer: micflan diff --git a/docs-mslearn/framework/manage/manage-finops.md b/docs-mslearn/framework/manage/manage-finops.md index 20a4602f9..bf5e136ec 100644 --- a/docs-mslearn/framework/manage/manage-finops.md +++ b/docs-mslearn/framework/manage/manage-finops.md @@ -3,7 +3,7 @@ title: Manage the FinOps practice description: Learn about the FinOps capabilities that help you establish your FinOps practice and drive organizational accountability. author: flanakin ms.author: micflan -ms.date: 02/24/2026 +ms.date: 04/01/2026 ms.topic: conceptual ms.service: finops ms.subservice: finops-learning-resources diff --git a/docs-mslearn/framework/manage/onboarding.md b/docs-mslearn/framework/manage/onboarding.md index 5ee559981..753e0f76f 100644 --- a/docs-mslearn/framework/manage/onboarding.md +++ b/docs-mslearn/framework/manage/onboarding.md @@ -3,7 +3,7 @@ title: Onboarding workloads description: This article helps you understand the onboarding workloads capability within the FinOps Framework and how to implement that in the Microsoft Cloud. author: flanakin ms.author: micflan -ms.date: 02/24/2026 +ms.date: 04/01/2026 ms.topic: concept-article ms.service: finops ms.subservice: finops-learning-resources diff --git a/docs-mslearn/framework/manage/operations.md b/docs-mslearn/framework/manage/operations.md index 035b6b13b..346b0a0d1 100644 --- a/docs-mslearn/framework/manage/operations.md +++ b/docs-mslearn/framework/manage/operations.md @@ -3,7 +3,7 @@ title: FinOps practice operations description: This article helps you understand the FinOps practice operations capability within the FinOps Framework and how to implement that in the Microsoft Cloud. author: flanakin ms.author: micflan -ms.date: 02/24/2026 +ms.date: 04/01/2026 ms.topic: concept-article ms.service: finops ms.subservice: finops-learning-resources diff --git a/docs-mslearn/framework/manage/tools-services.md b/docs-mslearn/framework/manage/tools-services.md index 2660935bc..d2c3bf349 100644 --- a/docs-mslearn/framework/manage/tools-services.md +++ b/docs-mslearn/framework/manage/tools-services.md @@ -3,7 +3,7 @@ title: FinOps tools and services description: This article helps you understand the FinOps tools and services capability within the FinOps Framework and how to implement that in the Microsoft Cloud. author: flanakin ms.author: micflan -ms.date: 02/24/2026 +ms.date: 04/01/2026 ms.topic: concept-article ms.service: finops ms.subservice: finops-learning-resources diff --git a/docs-mslearn/framework/optimize/architecting.md b/docs-mslearn/framework/optimize/architecting.md index 2e40da786..adecc0313 100644 --- a/docs-mslearn/framework/optimize/architecting.md +++ b/docs-mslearn/framework/optimize/architecting.md @@ -3,7 +3,7 @@ title: Architecting for cloud description: This article helps you understand the architecting for cloud capability within the FinOps Framework and how to implement that in the Microsoft Cloud. author: flanakin ms.author: micflan -ms.date: 02/24/2026 +ms.date: 04/01/2026 ms.topic: concept-article ms.service: finops ms.subservice: finops-learning-resources diff --git a/docs-mslearn/framework/optimize/licensing.md b/docs-mslearn/framework/optimize/licensing.md index 2b347a072..b6c72bfce 100644 --- a/docs-mslearn/framework/optimize/licensing.md +++ b/docs-mslearn/framework/optimize/licensing.md @@ -3,7 +3,7 @@ title: Licensing and SaaS description: This article helps you understand the licensing and SaaS capability within the FinOps Framework and how to implement that in the Microsoft Cloud. author: flanakin ms.author: micflan -ms.date: 02/24/2026 +ms.date: 04/01/2026 ms.topic: concept-article ms.service: finops ms.subservice: finops-learning-resources diff --git a/docs-mslearn/framework/optimize/optimize-cloud-usage-cost.md b/docs-mslearn/framework/optimize/optimize-cloud-usage-cost.md index ef94b6654..b2064dc11 100644 --- a/docs-mslearn/framework/optimize/optimize-cloud-usage-cost.md +++ b/docs-mslearn/framework/optimize/optimize-cloud-usage-cost.md @@ -3,7 +3,7 @@ title: Optimize usage and cost description: Learn about the FinOps capabilities that help you identify and implement the right mix of pricing models, services, and resources needed to meet business demands. author: flanakin ms.author: micflan -ms.date: 02/24/2026 +ms.date: 04/01/2026 ms.topic: conceptual ms.service: finops ms.subservice: finops-learning-resources diff --git a/docs-mslearn/framework/optimize/rates.md b/docs-mslearn/framework/optimize/rates.md index d29ee1535..4088ad56a 100644 --- a/docs-mslearn/framework/optimize/rates.md +++ b/docs-mslearn/framework/optimize/rates.md @@ -3,7 +3,7 @@ title: Rate optimization description: This article helps you understand the rate optimization capability within the FinOps Framework and how to implement that in the Microsoft Cloud. author: flanakin ms.author: micflan -ms.date: 02/24/2026 +ms.date: 04/01/2026 ms.topic: concept-article ms.service: finops ms.subservice: finops-learning-resources diff --git a/docs-mslearn/framework/optimize/sustainability.md b/docs-mslearn/framework/optimize/sustainability.md index ebdb087f4..b18df265b 100644 --- a/docs-mslearn/framework/optimize/sustainability.md +++ b/docs-mslearn/framework/optimize/sustainability.md @@ -3,7 +3,7 @@ title: Cloud sustainability description: This article helps you understand the cloud sustainability capability within the FinOps Framework and how to implement that in the Microsoft Cloud. author: flanakin ms.author: micflan -ms.date: 02/24/2026 +ms.date: 04/01/2026 ms.topic: concept-article ms.service: finops ms.subservice: finops-learning-resources diff --git a/docs-mslearn/framework/optimize/workloads.md b/docs-mslearn/framework/optimize/workloads.md index e8c02f3b0..4d585fd5a 100644 --- a/docs-mslearn/framework/optimize/workloads.md +++ b/docs-mslearn/framework/optimize/workloads.md @@ -3,7 +3,7 @@ title: Workload optimization description: This article helps you understand the Workload optimization capability within the FinOps Framework and how to implement that in the Microsoft Cloud. author: flanakin ms.author: micflan -ms.date: 02/24/2026 +ms.date: 04/01/2026 ms.topic: concept-article ms.service: finops ms.subservice: finops-learning-resources diff --git a/docs-mslearn/framework/quantify/benchmarking.md b/docs-mslearn/framework/quantify/benchmarking.md index 83c1e5b35..4db88a05f 100644 --- a/docs-mslearn/framework/quantify/benchmarking.md +++ b/docs-mslearn/framework/quantify/benchmarking.md @@ -3,7 +3,7 @@ title: FinOps benchmarking description: This article helps you understand the benchmarking capability within the FinOps Framework and how to implement that in the Microsoft Cloud. author: flanakin ms.author: micflan -ms.date: 02/24/2026 +ms.date: 04/01/2026 ms.topic: concept-article ms.service: finops ms.subservice: finops-learning-resources diff --git a/docs-mslearn/framework/quantify/budgeting.md b/docs-mslearn/framework/quantify/budgeting.md index efd3c575a..64d4c0668 100644 --- a/docs-mslearn/framework/quantify/budgeting.md +++ b/docs-mslearn/framework/quantify/budgeting.md @@ -3,7 +3,7 @@ title: Budgeting description: This article helps you understand the budgeting capability within the FinOps Framework and how to implement that in the Microsoft Cloud. author: flanakin ms.author: micflan -ms.date: 02/24/2026 +ms.date: 04/01/2026 ms.topic: concept-article ms.service: finops ms.subservice: finops-learning-resources diff --git a/docs-mslearn/framework/quantify/forecasting.md b/docs-mslearn/framework/quantify/forecasting.md index 8bc484aaf..dca1ef049 100644 --- a/docs-mslearn/framework/quantify/forecasting.md +++ b/docs-mslearn/framework/quantify/forecasting.md @@ -3,7 +3,7 @@ title: Forecasting description: This article helps you understand the forecasting capability within the FinOps Framework and how to implement that in the Microsoft Cloud. author: flanakin ms.author: micflan -ms.date: 02/24/2026 +ms.date: 04/01/2026 ms.topic: concept-article ms.service: finops ms.subservice: finops-learning-resources diff --git a/docs-mslearn/framework/quantify/planning.md b/docs-mslearn/framework/quantify/planning.md index 23d993141..abcc50b78 100644 --- a/docs-mslearn/framework/quantify/planning.md +++ b/docs-mslearn/framework/quantify/planning.md @@ -3,7 +3,7 @@ title: Planning and estimating description: This article helps you understand the planning and estimating capability within the FinOps Framework and how to implement that in the Microsoft Cloud. author: flanakin ms.author: micflan -ms.date: 02/24/2026 +ms.date: 04/01/2026 ms.topic: concept-article ms.service: finops ms.subservice: finops-learning-resources diff --git a/docs-mslearn/framework/quantify/quantify-business-value.md b/docs-mslearn/framework/quantify/quantify-business-value.md index 271e8ae1f..bb80c569a 100644 --- a/docs-mslearn/framework/quantify/quantify-business-value.md +++ b/docs-mslearn/framework/quantify/quantify-business-value.md @@ -3,7 +3,7 @@ title: Quantify business value description: Learn about the FinOps capabilities that help you measure product and cloud performance and map to organizational KPIs so you can make data-driven decisions with increased accuracy and velocity. author: flanakin ms.author: micflan -ms.date: 02/24/2026 +ms.date: 04/01/2026 ms.topic: conceptual ms.service: finops ms.subservice: finops-learning-resources diff --git a/docs-mslearn/framework/quantify/unit-economics.md b/docs-mslearn/framework/quantify/unit-economics.md index 7a0d0012f..02502bcaa 100644 --- a/docs-mslearn/framework/quantify/unit-economics.md +++ b/docs-mslearn/framework/quantify/unit-economics.md @@ -3,7 +3,7 @@ title: Unit economics description: This article helps you understand the unit economics capability within the FinOps Framework and how to implement that in the Microsoft Cloud. author: flanakin ms.author: micflan -ms.date: 02/24/2026 +ms.date: 04/01/2026 ms.topic: concept-article ms.service: finops ms.subservice: finops-learning-resources diff --git a/docs-mslearn/framework/understand/allocation.md b/docs-mslearn/framework/understand/allocation.md index efb58fde4..d721bc4de 100644 --- a/docs-mslearn/framework/understand/allocation.md +++ b/docs-mslearn/framework/understand/allocation.md @@ -3,7 +3,7 @@ title: Allocation description: This article helps you understand the Allocation capability within the FinOps Framework and how to implement that in the Microsoft Cloud. author: flanakin ms.author: micflan -ms.date: 02/24/2026 +ms.date: 04/01/2026 ms.topic: concept-article ms.service: finops ms.subservice: finops-learning-resources diff --git a/docs-mslearn/framework/understand/anomalies.md b/docs-mslearn/framework/understand/anomalies.md index 3c6e7b195..5d155649a 100644 --- a/docs-mslearn/framework/understand/anomalies.md +++ b/docs-mslearn/framework/understand/anomalies.md @@ -3,7 +3,7 @@ title: Anomaly management description: This article helps you understand the anomaly management capability within the FinOps Framework and how to implement that in the Microsoft Cloud. author: flanakin ms.author: micflan -ms.date: 02/24/2026 +ms.date: 04/01/2026 ms.topic: concept-article ms.service: finops ms.subservice: finops-learning-resources diff --git a/docs-mslearn/framework/understand/ingestion.md b/docs-mslearn/framework/understand/ingestion.md index 6cfaf788b..ca4f54025 100644 --- a/docs-mslearn/framework/understand/ingestion.md +++ b/docs-mslearn/framework/understand/ingestion.md @@ -3,7 +3,7 @@ title: Data ingestion and normalization description: This article helps you understand the data ingestion capability within the FinOps Framework and how to implement that in the Microsoft Cloud. author: flanakin ms.author: micflan -ms.date: 02/24/2026 +ms.date: 04/01/2026 ms.topic: concept-article ms.service: finops ms.subservice: finops-learning-resources diff --git a/docs-mslearn/framework/understand/reporting.md b/docs-mslearn/framework/understand/reporting.md index b2468d69b..d3fc304a3 100644 --- a/docs-mslearn/framework/understand/reporting.md +++ b/docs-mslearn/framework/understand/reporting.md @@ -3,7 +3,7 @@ title: Reporting and analytics description: This article helps you understand the reporting and analytics capability within the FinOps Framework and how to implement that in the Microsoft Cloud. author: flanakin ms.author: micflan -ms.date: 02/24/2026 +ms.date: 04/01/2026 ms.topic: concept-article ms.service: finops ms.subservice: finops-learning-resources diff --git a/docs-mslearn/framework/understand/understand-cloud-usage-cost.md b/docs-mslearn/framework/understand/understand-cloud-usage-cost.md index 02dac5e8d..33b7f724b 100644 --- a/docs-mslearn/framework/understand/understand-cloud-usage-cost.md +++ b/docs-mslearn/framework/understand/understand-cloud-usage-cost.md @@ -3,7 +3,7 @@ title: Understand usage and cost description: Learn about the FinOps capabilities that help you collect, normalize, analyze, and monitor cost, usage, and carbon across the organization. author: flanakin ms.author: micflan -ms.date: 02/24/2026 +ms.date: 04/01/2026 ms.topic: conceptual ms.service: finops ms.subservice: finops-learning-resources diff --git a/docs-mslearn/implementing-finops-guide.md b/docs-mslearn/implementing-finops-guide.md index e605f07fc..4e8c28d4f 100644 --- a/docs-mslearn/implementing-finops-guide.md +++ b/docs-mslearn/implementing-finops-guide.md @@ -3,7 +3,7 @@ title: Implementing FinOps guide description: Learn how FinOps can help you maintain business efficiency, empower new endeavors, and accelerate business growth through the cloud. author: flanakin ms.author: micflan -ms.date: 03/01/2026 +ms.date: 04/01/2026 ms.topic: concept-article ms.service: finops ms.subservice: finops-learning-resources diff --git a/docs-mslearn/overview.md b/docs-mslearn/overview.md index 522d10191..dc80cbcfc 100644 --- a/docs-mslearn/overview.md +++ b/docs-mslearn/overview.md @@ -3,7 +3,7 @@ title: What is FinOps? description: FinOps combines financial management principles with engineering and operations to provide organizations with a better understanding of their cloud spending. author: flanakin ms.author: micflan -ms.date: 03/03/2026 +ms.date: 04/01/2026 ms.topic: overview ms.service: finops ms.subservice: finops-learning-resources diff --git a/docs-mslearn/toolkit/alerts/configure-finops-alerts.md b/docs-mslearn/toolkit/alerts/configure-finops-alerts.md index 974a0c907..01f0cda5b 100644 --- a/docs-mslearn/toolkit/alerts/configure-finops-alerts.md +++ b/docs-mslearn/toolkit/alerts/configure-finops-alerts.md @@ -3,7 +3,7 @@ title: Configure FinOps alerts description: Learn how to configure and customize FinOps alerts to perform notifications and actions based on your organizational needs. author: flanakin ms.author: micflan -ms.date: 02/24/2026 +ms.date: 04/01/2026 ms.topic: concept-article ms.service: finops ms.subservice: finops-toolkit diff --git a/docs-mslearn/toolkit/alerts/finops-alerts-overview.md b/docs-mslearn/toolkit/alerts/finops-alerts-overview.md index ecfc3dc05..ebf5d7128 100644 --- a/docs-mslearn/toolkit/alerts/finops-alerts-overview.md +++ b/docs-mslearn/toolkit/alerts/finops-alerts-overview.md @@ -3,7 +3,7 @@ title: FinOps alerts overview description: FinOps alerts accelerate cost optimization efforts with scheduled notifications that continuously monitor your cloud environment. author: flanakin ms.author: micflan -ms.date: 02/24/2026 +ms.date: 04/01/2026 ms.topic: concept-article ms.service: finops ms.subservice: finops-toolkit diff --git a/docs-mslearn/toolkit/bicep-registry/modules.md b/docs-mslearn/toolkit/bicep-registry/modules.md index 124bdcd62..aacad3fbc 100644 --- a/docs-mslearn/toolkit/bicep-registry/modules.md +++ b/docs-mslearn/toolkit/bicep-registry/modules.md @@ -3,7 +3,7 @@ title: Bicep Registry modules for FinOps description: This article summarizes the Bicep modules available from the FinOps toolkit and provides guidance on how to reference them in your templates. author: flanakin ms.author: micflan -ms.date: 02/24/2026 +ms.date: 04/01/2026 ms.topic: concept-article ms.service: finops ms.subservice: finops-toolkit diff --git a/docs-mslearn/toolkit/bicep-registry/scheduled-actions.md b/docs-mslearn/toolkit/bicep-registry/scheduled-actions.md index c7ae28019..db2f74820 100644 --- a/docs-mslearn/toolkit/bicep-registry/scheduled-actions.md +++ b/docs-mslearn/toolkit/bicep-registry/scheduled-actions.md @@ -3,7 +3,7 @@ title: Cost Management scheduled action bicep modules description: This article describes the Cost Management scheduled actions Bicep Registry modules that help you send an email on a schedule or when an anomaly is detected. author: flanakin ms.author: micflan -ms.date: 02/24/2026 +ms.date: 04/01/2026 ms.topic: concept-article ms.service: finops ms.subservice: finops-toolkit diff --git a/docs-mslearn/toolkit/changelog.md b/docs-mslearn/toolkit/changelog.md index 09370c6e3..ff413afc9 100644 --- a/docs-mslearn/toolkit/changelog.md +++ b/docs-mslearn/toolkit/changelog.md @@ -3,7 +3,7 @@ title: FinOps toolkit changelog description: Review the latest features and enhancements in the FinOps toolkit, including updates to FinOps hubs, Power BI reports, and more. author: MSBrett ms.author: brettwil -ms.date: 03/25/2026 +ms.date: 04/04/2026 ms.topic: reference ms.service: finops ms.subservice: finops-toolkit @@ -12,20 +12,25 @@ ms.reviewer: brettwil --- - - + # FinOps toolkit changelog This article summarizes the features and enhancements in each release of the FinOps toolkit. - +> [!div class="nextstepaction"] +> [Give feedback](https://portal.azure.com/#view/HubsExtension/InProductFeedbackBlade/extensionName/FinOpsToolkit/cesQuestion/How%20easy%20or%20hard%20is%20it%20to%20use%20FinOps%20hubs%3F/cvaQuestion/How%20valuable%20are%20FinOps%20hubs%3F/surveyId/FTK/bladeName/Hubs/featureName/ConfigureAICopilotStudio) + + +If you're looking for something specific, vote for an existing or create a new idea. Share ideas with others to get more votes. We focus on ideas with the most votes. + + +> [!div class="nextstepaction"] +> [Vote on or suggest ideas](https://github.com/microsoft/finops-toolkit/issues?q=is%3Aissue%20is%3Aopen%20label%3A%22Tool%3A%20FinOps%20hubs%22%20sort%3Areactions-%2B1-desc) + + +
+ +## Related content + +Related FinOps hubs articles: + +- [Configure AI agents](configure-ai.md) +- [FinOps hubs overview](finops-hubs-overview.md) +- [Data model](data-model.md) + +Related products: + +- [Microsoft Copilot Studio](/microsoft-copilot-studio/fundamentals-what-is-copilot-studio) +- [Azure Data Explorer](/azure/data-explorer/) + +
diff --git a/docs-mslearn/toolkit/hubs/configure-ai.md b/docs-mslearn/toolkit/hubs/configure-ai.md index 15b43e8d4..76aae50ed 100644 --- a/docs-mslearn/toolkit/hubs/configure-ai.md +++ b/docs-mslearn/toolkit/hubs/configure-ai.md @@ -3,7 +3,7 @@ title: Configure AI agents for FinOps hubs description: Learn how to configure an AI agent to connect to your FinOps hub instance. author: flanakin ms.author: micflan -ms.date: 02/24/2026 +ms.date: 04/02/2026 ms.topic: how-to ms.service: finops ms.subservice: finops-toolkit @@ -59,6 +59,12 @@ For details about the Azure MCP server, see [Azure MCP on GitHub](https://github
+## Configure Microsoft Copilot Studio + +You can create a FinOps hub agent in Microsoft Copilot Studio that connects to your hub database via the Kusto Query MCP Server. This agent can be published to Microsoft Teams or Microsoft 365 Copilot, making cost insights available to your entire organization. + +For step-by-step instructions, see [Configure a FinOps hub agent in Microsoft Copilot Studio](configure-ai-copilot-studio.md). + ## Connect from other AI platforms FinOps hubs use [Model Context Protocol (MCP)](https://modelcontextprotocol.io/introduction) to connect to and query your data in Azure Data Explorer using the Azure MCP server. Besides GitHub Copilot, there are many popular [clients that support MCP servers](https://modelcontextprotocol.io/clients), like Claude, Continue, and more. While we have not tested instructions with other clients, you may be able to reuse some or all of the [AI instructions for FinOps hubs](https://github.com/microsoft/finops-toolkit/releases/latest/download/finops-hub-copilot.zip) with other clients. Try the instructions with clients you use and [create a change request](https://aka.ms/ftk/ideas) or [submit a pull request](https://github.com/microsoft/finops-toolkit/pulls) if you discover any gaps or improvements. diff --git a/docs-mslearn/toolkit/hubs/configure-dashboards.md b/docs-mslearn/toolkit/hubs/configure-dashboards.md index 3eb9784e5..4ff187bd2 100644 --- a/docs-mslearn/toolkit/hubs/configure-dashboards.md +++ b/docs-mslearn/toolkit/hubs/configure-dashboards.md @@ -3,7 +3,7 @@ title: Configure Data Explorer dashboards for FinOps hubs description: Deploy a pre-built Azure Data Explorer dashboard for FinOps hubs to start analyzing cost and usage for your accounts. author: flanakin ms.author: micflan -ms.date: 02/24/2026 +ms.date: 04/01/2026 ms.topic: how-to ms.service: finops ms.subservice: finops-toolkit diff --git a/docs-mslearn/toolkit/hubs/configure-recommendations.md b/docs-mslearn/toolkit/hubs/configure-recommendations.md new file mode 100644 index 000000000..aa7644600 --- /dev/null +++ b/docs-mslearn/toolkit/hubs/configure-recommendations.md @@ -0,0 +1,214 @@ +--- +title: Configure FinOps hubs recommendations +description: Learn about the recommendations available in FinOps hubs and how to add custom recommendations. +author: flanakin +ms.author: micflan +ms.date: 04/01/2026 +ms.topic: how-to +ms.service: finops +ms.subservice: finops-toolkit +ms.reviewer: micflan +# customer intent: As a FinOps hub user, I want to understand the recommendations available in FinOps hubs and how to add my own. +--- + + +# Configure FinOps hubs recommendations + + +FinOps hubs collect recommendations from multiple sources and ingest them into the [Recommendations managed dataset](data-model.md#recommendations-managed-dataset) alongside reservation recommendations from Cost Management exports. Recommendations are sourced from Azure Resource Graph using a configurable set of queries that pull Azure Advisor recommendations and identify various optimization scenarios based on resource configuration. Queries are managed in simple JSON files in storage, making it easy to add your own custom recommendations by uploading query files to hub storage. + +
+ +## Prerequisites + +Before you begin, you must have: + +- [Deployed a FinOps hub instance](finops-hubs-overview.md#create-a-new-hub). +- Assigned the **Reader** role to the Data Factory managed identity on the management groups or subscriptions you want to query. This permission must be configured separately from the FinOps hub deployment. + +
+ +## How recommendations are processed + +The recommendations pipeline runs daily and processes query files stored in the **config/queries** folder in hub storage: + +1. The **queries_DailySchedule** trigger runs once per day. +2. The **queries_ExecuteETL** pipeline iterates through all query files in the **config/queries** storage folder. +3. The **queries_ETL_ingestion** pipeline executes each query against Azure Resource Graph, deduplicates results, and saves data as parquet in the **ingestion/Recommendations** folder. +4. If using Azure Data Explorer, data is ingested into the `Recommendations_raw` table and transformed using the `Recommendations_transform_v1_2()` function. + +Hubs recommendations are combined with reservation recommendations from Cost Management exports in the same [Recommendations managed dataset](data-model.md#recommendations-managed-dataset). You can distinguish between sources using the `x_SourceType` column. + +
+ +## Built-in recommendations + +FinOps hubs include the following recommendations. Most are enabled by default. Optional recommendations may generate noise for organizations where they don't apply and can be enabled during deployment via the specified template parameter. + +### Compute + +- **Virtual Machines** + - [Deallocate stopped VMs](../../best-practices/compute.md#deallocate-virtual-machines). + - [Migrate to managed disks](../../best-practices/compute.md#migrate-to-managed-disks). + - Optional: [Use Azure Hybrid Benefit for Windows VMs](../../best-practices/compute.md#use-azure-hybrid-benefit-for-windows-vms). Enabled via the `enableAHBRecommendations` option. +- **SQL Virtual Machines** + - Optional: [Use Azure Hybrid Benefit for SQL VMs](../../best-practices/compute.md#use-azure-hybrid-benefit-for-sql-vms). Enabled via the `enableAHBRecommendations` option. +- **Azure Kubernetes Service** + - Optional: [Use Spot VMs for AKS clusters](../../best-practices/compute.md#use-spot-vms-for-aks-clusters). Enabled via the `enableSpotRecommendations` option. + +### Databases + +- **Azure SQL Database** + - [Remove unused elastic pools](../../best-practices/databases.md#remove-unused-elastic-pools) + +### Management and Governance + +- **Azure Advisor** + - [Review Azure Advisor cost recommendations](../../best-practices/general.md#review-azure-advisor-cost-recommendations) + +### Networking + +- **Application Gateway** + - [Remove idle application gateways](../../best-practices/networking.md#remove-idle-application-gateways). + - [Upgrade classic application gateways](../../best-practices/networking.md#upgrade-classic-application-gateways) +- **DDoS Protection** + - [Remove unassociated DDoS plans](../../best-practices/networking.md#remove-unassociated-ddos-protection-plans) +- **ExpressRoute** + - [Remove unprovisioned ExpressRoute circuits](../../best-practices/networking.md#remove-unprovisioned-expressroute-circuits) +- **Load Balancer** + - [Remove idle load balancers](../../best-practices/networking.md#remove-idle-load-balancers). + - [Upgrade Basic load balancers](../../best-practices/networking.md#upgrade-basic-load-balancers) +- **NAT Gateway** + - [Remove orphaned NAT gateways](../../best-practices/networking.md#remove-orphaned-nat-gateways) +- **Network Interfaces** + - [Remove unattached NICs](../../best-practices/networking.md#remove-unattached-network-interfaces) +- **Network Security Groups** + - [Remove empty NSGs](../../best-practices/networking.md#remove-empty-network-security-groups) +- **Public IP Addresses** + - [Remove unattached public IPs](../../best-practices/networking.md#remove-idle-public-ip-addresses). + - [Upgrade Basic public IPs](../../best-practices/networking.md#upgrade-basic-public-ips) +- **VPN Gateway** + - [Remove idle VNet gateways](../../best-practices/networking.md#remove-idle-vnet-gateways) + +### Storage + +- **Managed Disks** + - [Downgrade premium snapshots](../../best-practices/storage.md#downgrade-premium-snapshots). + - [Remove unattached disks](../../best-practices/storage.md#remove-unattached-disks) +- **Storage Accounts** + - [Upgrade legacy storage accounts](../../best-practices/storage.md#upgrade-legacy-storage-accounts) + +### Web + +- **App Service** + - [Remove empty App Service plans](../../best-practices/web.md#remove-empty-app-service-plans) + +To disable a specific default recommendation, delete its query file from the **config/queries** folder in hub storage. The pipeline only processes query files that are present. + +
+ +## Add custom recommendations + +You can add custom recommendations by uploading query files to the **config/queries** folder in hub storage. The pipeline picks up new query files automatically on the next daily run. + +### File naming convention + +Name query files using the `{dataset}-{provider}-{type}.json` format: + +- **Dataset** — The target dataset (for example, `Recommendations`). +- **Provider** — The provider of the service data is for (for example, `Microsoft`, `Contoso`). +- **Type** — The recommendation type identifier using PascalCase (for example, `StoppedVMs`, `IdleCosmosDB`). + +For example: `Recommendations-Contoso-IdleCosmosDB.json` + +### Query file format + +Each query file is a JSON file with the following properties: + +```json +{ + "dataset": "Recommendations", + "provider": "Microsoft", + "query": "", + "queryEngine": "ResourceGraph", + "scope": "Tenant", + "source": "", + "type": "", + "version": "1.0" +} +``` + +| Property | Description | +| ------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `dataset` | Must be `"Recommendations"`. | +| `provider` | Provider name (for example, `"Microsoft"`). | +| `query` | The Azure Resource Graph query to execute, on a single line. | +| `queryEngine` | Must be `"ResourceGraph"`. | +| `scope` | Query scope. Use `"Tenant"` to query all subscriptions the Data Factory managed identity has access to within the tenant. Cross-tenant queries aren't supported but resources delegated via Azure Lighthouse are included in tenant-scope queries. | +| `source` | Descriptive name for the recommendation source (for example, `"Azure Advisor"` or `"FinOps hubs"`). | +| `type` | Programmatic identifier for this recommendation type. Use a `{provider}-{name}` format with alphanumeric characters and hyphens only (for example, `"Contoso-IdleCosmosDB"`). This value is used as part of the output file name. | +| `version` | Schema version. Use `"1.0"`. | + +### Required output columns + +Your query must return the following columns: + +| Column | Description | +| ----------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `ResourceId` | Resource ID (lowercase). | +| `ResourceName` | Resource name (lowercase). | +| `SubAccountId` | Subscription ID. | +| `SubAccountName` | Subscription name. Join with `resourcecontainers` to populate this. | +| `x_RecommendationCategory` | Recommendation category. Use `"Cost"`, `"HighAvailability"`, `"OperationalExcellence"`, `"Performance"`, or `"Security"`. | +| `x_RecommendationDate` | Recommendation date (use `now()` for point-in-time queries). | +| `x_RecommendationDescription` | Short description of the issue. | +| `x_RecommendationDetails` | JSON string with additional properties. Include `x_RecommendationProvider`, `x_RecommendationSolution`, `x_RecommendationTypeId`, and `x_ResourceType` along with any custom properties specific to the recommendation. | +| `x_RecommendationId` | Unique identifier for the recommendation (for example, resource ID + suffix). | +| `x_ResourceGroupName` | Resource group name (lowercase). | + +### Tips for writing queries + +- To populate the subscription name, join with `resourcecontainers` at the end of your query: + + ```kusto + | join kind=leftouter ( + resourcecontainers + | where type == 'microsoft.resources/subscriptions' + | project SubAccountName=name, SubAccountId=subscriptionId + ) on SubAccountId + | project-away SubAccountId1 + ``` + +- Generate `x_RecommendationId` by combining the resource ID with a descriptive suffix (for example, `strcat(tolower(id), '-idle')`). +- Build `x_RecommendationDetails` using `tostring(bag_pack(...))` to produce a JSON string. Wrapping with `tostring()` is required because the data pipeline can't serialize dynamic objects to parquet; the value must be a string. You can also use `strcat()` to build a JSON string manually, but `bag_pack()` is recommended because it handles escaping automatically. +- Include `x_RecommendationTypeId` as a stable GUID to uniquely identify the recommendation type across runs. + +For examples, review the built-in query files in the [FinOps toolkit source code](https://github.com/microsoft/finops-toolkit/tree/dev/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Recommendations/queries). + +
+ +## Give feedback + +Let us know how we're doing with a quick review. We use these reviews to improve and expand FinOps tools and resources. + + +> [!div class="nextstepaction"] +> [Give feedback](https://portal.azure.com/#view/HubsExtension/InProductFeedbackBlade/extensionName/FinOpsToolkit/cesQuestion/How%20easy%20or%20hard%20is%20it%20to%20use%20FinOps%20hubs%3F/cvaQuestion/How%20valuable%20are%20FinOps%20hubs%3F/surveyId/FTK/bladeName/Hubs/featureName/Recommendations) + + +If you're looking for something specific, vote for an existing or create a new idea. Share ideas with others to get more votes. We focus on ideas with the most votes. + + +> [!div class="nextstepaction"] +> [Vote on or suggest ideas](https://github.com/microsoft/finops-toolkit/issues?q=is%3Aissue%20is%3Aopen%20sort%3Areactions-%2B1-desc) + + +
+ +## Related content + +- [Recommendations managed dataset](data-model.md#recommendations-managed-dataset) +- [How data is processed in FinOps hubs](data-processing.md) +- [Best practices library](../../best-practices/library.md) + +
diff --git a/docs-mslearn/toolkit/hubs/configure-remote-hubs.md b/docs-mslearn/toolkit/hubs/configure-remote-hubs.md index 445f47792..7ab7c8f85 100644 --- a/docs-mslearn/toolkit/hubs/configure-remote-hubs.md +++ b/docs-mslearn/toolkit/hubs/configure-remote-hubs.md @@ -2,7 +2,7 @@ ms.service: finops ms.author: flanakin author: flanakin -ms.date: 11/01/2024 +ms.date: 04/01/2026 ms.topic: how-to title: Configure remote hubs description: Learn how to configure FinOps hubs to collect cost data across multiple Azure tenants and clouds using remote hub functionality. diff --git a/docs-mslearn/toolkit/hubs/configure-scopes.md b/docs-mslearn/toolkit/hubs/configure-scopes.md index 6ca5528ac..31d8ef6b1 100644 --- a/docs-mslearn/toolkit/hubs/configure-scopes.md +++ b/docs-mslearn/toolkit/hubs/configure-scopes.md @@ -3,7 +3,7 @@ title: Configure scopes for FinOps hubs description: Connect FinOps hubs to billing accounts and subscriptions by configuring Cost Management exports manually or give FinOps hubs access to manage exports for you. author: flanakin ms.author: micflan -ms.date: 03/10/2026 +ms.date: 04/01/2026 ms.topic: how-to ms.service: finops ms.subservice: finops-toolkit diff --git a/docs-mslearn/toolkit/hubs/data-model.md b/docs-mslearn/toolkit/hubs/data-model.md index 1f3818b2f..e06a19519 100644 --- a/docs-mslearn/toolkit/hubs/data-model.md +++ b/docs-mslearn/toolkit/hubs/data-model.md @@ -3,7 +3,7 @@ title: FinOps hubs data model description: Learn about the tables and functions available in FinOps hubs to build your own queries, reports, and dashboards. author: flanakin ms.author: micflan -ms.date: 02/24/2026 +ms.date: 04/01/2026 ms.topic: reference ms.service: finops ms.subservice: finops-toolkit @@ -714,7 +714,13 @@ The following are provided for backwards compatibility: - **Recommendations_final_v1_0** table in the **Ingestion** database to host data ingested with FinOps hubs 0.7-0.11. - **Recommendations_v1_0()** function in the **Hub** database to convert all data to FOCUS 1.0. -The **Recommendations_raw** table supports Microsoft Cost Management reservation recommendation export schemas for EA and MCA accounts. Data is transformed into a FOCUS-aligned dataset when ingested into the final table. This dataset does not explicitly support other clouds. +The **Recommendations_raw** table supports the following data sources: + +- Microsoft Cost Management reservation recommendation export schemas for EA and MCA accounts. +- Azure Advisor cost recommendations via Azure Resource Graph (HubsRecommendations). +- Custom recommendations from Azure Resource Graph queries (HubsRecommendations). + +Data is transformed into a FOCUS-aligned dataset when ingested into the final table. This dataset does not explicitly support other clouds. Columns in the **Recommendations** managed dataset include: diff --git a/docs-mslearn/toolkit/hubs/data-processing.md b/docs-mslearn/toolkit/hubs/data-processing.md index 6c1c7eb47..9c3e1cabb 100644 --- a/docs-mslearn/toolkit/hubs/data-processing.md +++ b/docs-mslearn/toolkit/hubs/data-processing.md @@ -3,7 +3,7 @@ title: FinOps hubs data processing description: Learn how FinOps hubs process data, including scope setup, data normalization, and optimization, to enhance cost management and analysis. author: flanakin ms.author: micflan -ms.date: 03/23/2026 +ms.date: 04/01/2026 ms.topic: concept-article ms.service: finops ms.subservice: finops-toolkit @@ -54,7 +54,7 @@ The following diagram depicts the end-to-end data ingestion process within FinOp 4. After ingestion, the **ingestion_ETL_dataExplorer** pipeline performs some cleanup, including purging data in the final table that is past the data retention period. - As of 0.7, Data Explorer applies data retention in raw tables while data retention in final tables is applied by the ingestion pipeline. If data ingestion stops, historical data isn't purged. - Data retention can be configured during the template deployment or manually in the **config/settings.json** file in storage. -6. (Optional) The **queries_DailySchedule** trigger runs the **queries_ExecuteETL** pipeline once per day to query Azure Resource Graph for additional recommendations and saves results to the **ingestion/Recommendations** folder. [Learn more](#recommendation-data-transforms). +6. (Optional) The **queries_DailySchedule** trigger runs the **queries_ExecuteETL** pipeline once per day to query Azure Resource Graph for additional recommendations and saves results to the **ingestion/Recommendations** folder. [Learn more](#azure-resource-graph-recommendations). 7. Reports and other tools like Power BI read data from Data Explorer or the **ingestion** container. - Data in Data Explorer can be read from the **Hub** database. - Use the `{dataset}()` function to use the latest schema. @@ -69,6 +69,35 @@ The following diagram depicts the end-to-end data ingestion process within FinOp
+## Azure Resource Graph recommendations + +When FinOps hubs are deployed with Azure Data Explorer or Microsoft Fabric, an additional pipeline runs daily to query Azure Resource Graph for cost optimization recommendations. This includes Azure Advisor cost recommendations and custom queries for common cost optimization scenarios. + +The following diagram depicts the HubsRecommendations data flow: + +1. The **queries_DailySchedule** trigger runs once per day. +2. The **queries_ExecuteETL** pipeline iterates through all query files in the **config/queries** storage folder. +3. The **queries_ETL_ingestion** pipeline executes each query against Azure Resource Graph, deduplicates results, and saves data as parquet in the **ingestion/Recommendations** folder. +4. (If using Azure Data Explorer) Data is ingested into the `Recommendations_raw` table and transformed using the `Recommendations_transform_v1_2()` function. + +HubsRecommendations includes the following query types: + +- **Azure Advisor cost recommendations** - Cost optimization recommendations from Azure Advisor. +- **Backendless Application Gateways** - Application Gateways without backend pools. +- **Backendless Load Balancers** - Load Balancers without backend pools. +- **Empty SQL Elastic Pools** - SQL Elastic Pools with no associated databases. +- **Non-Spot AKS Clusters** - AKS clusters with autoscaling but not using Spot VMs. +- **SQL VMs without Azure Hybrid Benefit** - SQL virtual machines not leveraging Azure Hybrid Benefit. +- **Stopped VMs** - Virtual machines that are stopped but not deallocated. +- **Unattached Disks** - Unattached (orphaned) managed disks. +- **Unattached Public IPs** - Unattached static public IP addresses. +- **VMs without Azure Hybrid Benefit** - Windows VMs not leveraging Azure Hybrid Benefit. + +> [!NOTE] +> The Data Factory managed identity requires **Reader** role on management groups or subscriptions to execute Resource Graph queries. This permission must be configured separately from the FinOps hub deployment. + +
+ ## About Data Explorer ingestion When data is ingested into Data Explorer, the `{dataset}_transform_v1_0()` functions apply transform rules in the **Ingestion** database. Each dataset has a different set of transform rules covered in the following sections. @@ -194,6 +223,7 @@ Transforms: Supported datasets: - Microsoft ReservationRecommendations: `2023-05-01` (EA and MCA) +- HubsRecommendations: `1.0` (Azure Advisor and Azure Resource Graph) Transforms: @@ -202,6 +232,7 @@ Transforms: - Includes enforcing EA and MCA column name consistency. - Doesn't change the underlying values, which may differ across EA and MCA. - Add `x_SourceName`, `x_SourceProvider`, `x_SourceType`, and `x_SourceVersion` to identify the original ingested dataset. + - Preserve incoming `x_RecommendationDetails` from HubsRecommendations queries (contains recommendation metadata like provider, solution, type ID, and resource type). ### Transaction data transforms diff --git a/docs-mslearn/toolkit/hubs/deploy.md b/docs-mslearn/toolkit/hubs/deploy.md index 950fcf5a8..ac1300b91 100644 --- a/docs-mslearn/toolkit/hubs/deploy.md +++ b/docs-mslearn/toolkit/hubs/deploy.md @@ -3,7 +3,7 @@ title: How to create and update FinOps hubs description: This tutorial helps you create a new or update an existing FinOps hubs instance in Azure or Microsoft Fabric. author: flanakin ms.author: micflan -ms.date: 02/24/2026 +ms.date: 04/01/2026 ms.topic: tutorial ms.service: finops ms.subservice: finops-toolkit diff --git a/docs-mslearn/toolkit/hubs/finops-hubs-overview.md b/docs-mslearn/toolkit/hubs/finops-hubs-overview.md index d6404479e..5292e0c9d 100644 --- a/docs-mslearn/toolkit/hubs/finops-hubs-overview.md +++ b/docs-mslearn/toolkit/hubs/finops-hubs-overview.md @@ -3,7 +3,7 @@ title: FinOps hubs overview description: FinOps hubs provide a reliable platform for cost analytics, insights, and optimization, supporting large accounts and organizations. author: flanakin ms.author: micflan -ms.date: 02/24/2026 +ms.date: 04/01/2026 ms.topic: concept-article ms.service: finops ms.subservice: finops-toolkit diff --git a/docs-mslearn/toolkit/hubs/private-networking.md b/docs-mslearn/toolkit/hubs/private-networking.md index 313b52e03..d328c8d6e 100644 --- a/docs-mslearn/toolkit/hubs/private-networking.md +++ b/docs-mslearn/toolkit/hubs/private-networking.md @@ -3,7 +3,7 @@ title: Configure private networking in FinOps hubs description: Learn about data access options with FinOps hubs and how to configure secure access to your data with private endpoints. author: flanakin ms.author: micflan -ms.date: 02/24/2026 +ms.date: 04/01/2026 ms.topic: how-to ms.service: finops ms.reviewer: micflan diff --git a/docs-mslearn/toolkit/hubs/savings-calculations.md b/docs-mslearn/toolkit/hubs/savings-calculations.md index e10eff146..d47032947 100644 --- a/docs-mslearn/toolkit/hubs/savings-calculations.md +++ b/docs-mslearn/toolkit/hubs/savings-calculations.md @@ -3,7 +3,7 @@ title: Understanding savings calculations in FinOps toolkit description: Learn how savings values are calculated and displayed in FinOps toolkit reports, including negative savings and missing price scenarios. author: flanakin ms.author: micflan -ms.date: 02/24/2026 +ms.date: 04/01/2026 ms.topic: concept-article ms.service: finops ms.subservice: finops-toolkit diff --git a/docs-mslearn/toolkit/hubs/template.md b/docs-mslearn/toolkit/hubs/template.md index 027d7a8f1..e0965d4a4 100644 --- a/docs-mslearn/toolkit/hubs/template.md +++ b/docs-mslearn/toolkit/hubs/template.md @@ -3,7 +3,7 @@ title: FinOps hub template description: Learn about what's included in the FinOps hub template including parameters, resources, and outputs. author: flanakin ms.author: micflan -ms.date: 02/24/2026 +ms.date: 04/01/2026 ms.topic: concept-article ms.service: finops ms.subservice: finops-toolkit @@ -11,8 +11,9 @@ ms.reviewer: micflan #customer intent: As a FinOps user, I want to understand what FinOps hubs are so that I can use them in my organization. --- - + # FinOps hub template + This document provides a detailed summary of what's included in the FinOps hubs deployment template. You can use this as a guide for tuning your deployment or to inform customizations you can make to the template to meet your organizational needs. This document explains the required prerequisites to deploy the template, input parameters you can customize, resources that will be deployed, and the template outputs. Template outputs can be used to connect to your hub instances in Power BI, Data Explorer, or other tools. @@ -22,8 +23,10 @@ FinOps hubs includes many resources to offer a secure and scalable FinOps platfo - Storage account (Data Lake Storage Gen2) as a staging area for data ingestion. - Data Factory instance to manage data ingestion and cleanup. + > [!IMPORTANT] > To use the template, you need to create Cost Management exports to publish data to the `msexports` container in the included storage account. For more information, see [Create a new hub](finops-hubs-overview.md#create-a-new-hub). +
@@ -31,42 +34,44 @@ FinOps hubs includes many resources to offer a secure and scalable FinOps platfo Ensure the following prerequisites are met before you deploy the template: + - You must have the following permissions to create the [deployed resources](#resources). - | Resource | Minimum Azure RBAC | - | ---------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | - | Deploy and configure Data Factory¹ | [Data Factory Contributor](/azure/role-based-access-control/built-in-roles#data-factory-contributor) | - | Deploy Key Vault (remote hub only)¹ | [Key Vault Contributor](/azure/role-based-access-control/built-in-roles#key-vault-contributor) | - | Configure Key Vault secrets (remote hub only)¹ | [Key Vault Administrator](/azure/role-based-access-control/built-in-roles#key-vault-administrator) | - | Create managed identity¹ | [Managed Identity Contributor](/azure/role-based-access-control/built-in-roles#managed-identity-contributor) | - | Deploy and configure storage¹ | [Storage Account Contributor](/azure/role-based-access-control/built-in-roles#storage-account-contributor) | - | Assign managed identity to resources¹ | [Managed Identity Operator](/azure/role-based-access-control/built-in-roles#managed-identity-operator) | - | Create deployment scripts¹ | Custom role containing only the `Microsoft.Resources/deploymentScripts/write` and `Microsoft.ContainerInstance/containerGroups/write` permissions as allowed actions or, alternatively, [Contributor](/azure/role-based-access-control/built-in-roles#contributor), which includes these permissions and all the above roles | - | Assign permissions to managed identities¹ | [Role Based Access Control Administrator](/azure/role-based-access-control/built-in-roles#role-based-access-control-administrator) or, alternatively, [Owner](/azure/role-based-access-control/built-in-roles#owner), which includes this role and all the above roles | - | Create a subscription or resource group cost export² | [Cost Management Contributor](/azure/role-based-access-control/built-in-roles#cost-management-contributor) | - | Create an EA billing cost export² | Enterprise Reader, Department Reader, or Enrollment Account Owner ([Learn more](/azure/cost-management-billing/manage/understand-ea-roles)) | - | Create an MCA billing cost export² | [Contributor](/azure/cost-management-billing/manage/understand-mca-roles) | - | Read blob data in storage³ | [Storage Blob Data Contributor](/azure/role-based-access-control/built-in-roles#storage-blob-data-contributor) | - - _¹ It's sufficient to assign hubs resources deployment permissions on the resource group scope._
- _² Cost Management permissions must be assigned on the scope where you want to export your costs from._
- _³ Blob data permissions are required to access exported cost data from Power BI or other client tools._
+ | Resource | Minimum Azure RBAC | + | ---------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | + | Deploy and configure Data Factory¹ | [Data Factory Contributor](/azure/role-based-access-control/built-in-roles#data-factory-contributor) | + | Deploy Key Vault (remote hub only)¹ | [Key Vault Contributor](/azure/role-based-access-control/built-in-roles#key-vault-contributor) | + | Configure Key Vault secrets (remote hub only)¹ | [Key Vault Administrator](/azure/role-based-access-control/built-in-roles#key-vault-administrator) | + | Create managed identity¹ | [Managed Identity Contributor](/azure/role-based-access-control/built-in-roles#managed-identity-contributor) | + | Deploy and configure storage¹ | [Storage Account Contributor](/azure/role-based-access-control/built-in-roles#storage-account-contributor) | + | Assign managed identity to resources¹ | [Managed Identity Operator](/azure/role-based-access-control/built-in-roles#managed-identity-operator) | + | Create deployment scripts¹ | Custom role containing only the `Microsoft.Resources/deploymentScripts/write` and `Microsoft.ContainerInstance/containerGroups/write` permissions as allowed actions or, alternatively, [Contributor](/azure/role-based-access-control/built-in-roles#contributor), which includes these permissions and all the above roles | + | Assign permissions to managed identities¹ | [Role Based Access Control Administrator](/azure/role-based-access-control/built-in-roles#role-based-access-control-administrator) or, alternatively, [Owner](/azure/role-based-access-control/built-in-roles#owner), which includes this role and all the above roles | + | Create a subscription or resource group cost export² | [Cost Management Contributor](/azure/role-based-access-control/built-in-roles#cost-management-contributor) | + | Create an EA billing cost export² | Enterprise Reader, Department Reader, or Enrollment Account Owner ([Learn more](/azure/cost-management-billing/manage/understand-ea-roles)) | + | Create an MCA billing cost export² | [Contributor](/azure/cost-management-billing/manage/understand-mca-roles) | + | Read blob data in storage³ | [Storage Blob Data Contributor](/azure/role-based-access-control/built-in-roles#storage-blob-data-contributor) | + + _¹ It's sufficient to assign hubs resources deployment permissions on the resource group scope._
+ _² Cost Management permissions must be assigned on the scope where you want to export your costs from._
+ _³ Blob data permissions are required to access exported cost data from Power BI or other client tools._
- You must have permissions to assign the following roles to managed identities as part of the deployment: - | Azure RBAC role | Notes | - | -------------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------- | - | [Data Factory Contributor](/azure/role-based-access-control/built-in-roles#data-factory-contributor) | Assigned to the deployment trigger manager identity to auto-start Data Factory triggers. | - | [Reader](/azure/role-based-access-control/built-in-roles#reader) | Assigned to Data Factory to manage data in storage. | - | [Storage Account Contributor](/azure/role-based-access-control/built-in-roles#storage-account-contributor) | Assigned to Data Factory to manage data in storage. | - | [Storage Blob Data Contributor](/azure/role-based-access-control/built-in-roles#storage-blob-data-contributor) | Assigned to Data Factory and Data Explorer to manage data in storage. | - | [Storage File Data Privileged Contributor](/azure/role-based-access-control/built-in-roles/storage#storage-file-data-privileged-contributor) | Assigned to the deployment file upload identity that uploads files to the config container. | - | [User Access Administrator](/azure/role-based-access-control/built-in-roles#user-access-administrator) | Assigned to Data Factory to manage data in storage. Not applied when **enableManagedExports** is disabled. | + | Azure RBAC role | Notes | + | -------------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------- | + | [Data Factory Contributor](/azure/role-based-access-control/built-in-roles#data-factory-contributor) | Assigned to the deployment trigger manager identity to auto-start Data Factory triggers. | + | [Reader](/azure/role-based-access-control/built-in-roles#reader) | Assigned to Data Factory to manage data in storage. | + | [Storage Account Contributor](/azure/role-based-access-control/built-in-roles#storage-account-contributor) | Assigned to Data Factory to manage data in storage. | + | [Storage Blob Data Contributor](/azure/role-based-access-control/built-in-roles#storage-blob-data-contributor) | Assigned to Data Factory and Data Explorer to manage data in storage. | + | [Storage File Data Privileged Contributor](/azure/role-based-access-control/built-in-roles/storage#storage-file-data-privileged-contributor) | Assigned to the deployment file upload identity that uploads files to the config container. | + | [User Access Administrator](/azure/role-based-access-control/built-in-roles#user-access-administrator) | Assigned to Data Factory to manage data in storage. Not applied when **enableManagedExports** is disabled. | - The Microsoft.EventGrid resource provider must be registered in your subscription. For more information, see [Register a resource provider](/azure/azure-resource-manager/management/resource-providers-and-types#register-resource-provider). - > [!IMPORTANT] - > If you forget this step, the deployment will succeed, but the pipeline trigger will not be started and data will not be ready. For more information, see [Troubleshooting Power BI reports](../help/troubleshooting.md). + > [!IMPORTANT] + > If you forget this step, the deployment will succeed, but the pipeline trigger will not be started and data will not be ready. For more information, see [Troubleshooting Power BI reports](../help/troubleshooting.md). +
@@ -93,6 +98,9 @@ Here are the parameters you can use to customize the deployment: | **remoteHubStorageUri** | String | Optional. Data Lake storage endpoint from the remote (primary) hub storage account. Used for cross-tenant cost data collection where this hub sends processed data to a central hub. Example: `https://primaryhub.dfs.core.windows.net/` | | | **remoteHubStorageKey** | String | Optional. Storage account access key for the remote (primary) hub. Used with remoteHubStorageUri for cross-tenant scenarios. Must be kept secure as it provides full storage access. | | | **enableManagedExports** | Bool | Optional. Enable managed exports where your FinOps hub instance will create and run Cost Management exports on your behalf. Not supported for Microsoft Customer Agreement (MCA) billing profiles. Requires the ability to grant User Access Administrator role to FinOps hubs, which is required to create Cost Management exports. | True | +| **enableRecommendations** | Bool | Optional. Enable recommendations ingested from Azure Resource Graph based on configurable queries. The Data Factory managed identity requires Reader role on management groups or subscriptions to execute Resource Graph queries. | True | +| **enableAHBRecommendations** | Bool | Optional. Enable Azure Hybrid Benefit recommendations that flag VMs and SQL VMs without Azure Hybrid Benefit enabled. May generate noise if your organization does not have on-premises licenses. Requires enableRecommendations. | False | +| **enableSpotRecommendations** | Bool | Optional. Enable non-Spot AKS cluster recommendations that flag AKS clusters with autoscaling but not using Spot VMs. May generate noise since Spot VMs are only appropriate for interruptible workloads. Requires enableRecommendations. | False | | **enablePublicAccess** | Bool | Optional. Disable public access to the data lake (storage firewall). | True | | **virtualNetworkAddressPrefix** | String | Optional. IP Address range for the private virtual network used by FinOps hubs. Accepts any subnet size from `/8` to `/26` with a minimum of `/26` required. `/26` is recommended to avoid wasting IPs unless you need additional address space for services like Power BI VNet Data Gateway. Internally, the following subnets will be created: `/28` for private endpoints, another `/28` subnet for temporary deployment scripts (container instances), and `/27` for Azure Data Explorer, if enabled. | '10.20.30.0/26' | @@ -189,18 +197,18 @@ Here are the outputs generated by the deployment: | Output | Type | Description | Value | | ------ | ---- | ----------- || -| **name** | String | Name of the resource group. | -| **location** | String | Azure resource location resources were deployed to. | -| **dataFactoryName** | String | Name of the Data Factory. | -| **storageAccountId** | String | Resource ID of the deployed storage account. | -| **storageAccountName** | String | Name of the storage account created for the hub instance. This must be used when connecting FinOps toolkit Power BI reports to your data. | -| **storageUrlForPowerBI** | String | URL to use when connecting custom Power BI reports to your data. | -| **clusterId** | String | Resource ID of the Data Explorer cluster. | -| **clusterUri** | String | URI of the Data Explorer cluster. | -| **ingestionDbName** | String | Name of the Data Explorer database used for ingesting data. | -| **hubDbName** | String | Name of the Data Explorer database used for querying data. | -| **managedIdentityId** | String | Object ID of the Data Factory managed identity. This will be needed when configuring managed exports. | -| **managedIdentityTenantId** | String | Azure AD tenant ID. This will be needed when configuring managed exports. | +| **name** | String | Name of the resource group. | +| **location** | String | Azure resource location resources were deployed to. | +| **dataFactoryName** | String | Name of the Data Factory. | +| **storageAccountId** | String | Resource ID of the deployed storage account. | +| **storageAccountName** | String | Name of the storage account created for the hub instance. This must be used when connecting FinOps toolkit Power BI reports to your data. | +| **storageUrlForPowerBI** | String | URL to use when connecting custom Power BI reports to your data. | +| **clusterId** | String | Resource ID of the Data Explorer cluster. | +| **clusterUri** | String | URI of the Data Explorer cluster. | +| **ingestionDbName** | String | Name of the Data Explorer database used for ingesting data. | +| **hubDbName** | String | Name of the Data Explorer database used for querying data. | +| **managedIdentityId** | String | Object ID of the Data Factory managed identity. This will be needed when configuring managed exports. | +| **managedIdentityTenantId** | String | Azure AD tenant ID. This will be needed when configuring managed exports. |
@@ -208,20 +216,26 @@ Here are the outputs generated by the deployment: Let us know how we're doing with a quick review. We use these reviews to improve and expand FinOps tools and resources. + > [!div class="nextstepaction"] > [Give feedback](https://portal.azure.com/#view/HubsExtension/InProductFeedbackBlade/extensionName/FinOpsToolkit/cesQuestion/How%20easy%20or%20hard%20is%20it%20to%20use%20FinOps%20hubs%3F/cvaQuestion/How%20valuable%20are%20FinOps%20hubs%3F/surveyId/FTK/bladeName/Hubs/featureName/Template) + If you're looking for something specific, vote for an existing or create a new idea. Share ideas with others to get more votes. We focus on ideas with the most votes. + > [!div class="nextstepaction"] > [Vote on or suggest ideas](https://github.com/microsoft/finops-toolkit/issues?q=is%3Aissue%20is%3Aopen%20label%3A%22Tool%3A%20FinOps%20hubs%22%20sort%3A"reactions-%2B1-desc") +
## Related content + > [!div class="nextstepaction"] > [Deploy FinOps hubs](finops-hubs-overview.md#create-a-new-hub) + [Learn more](finops-hubs-overview.md#why-finops-hubs) diff --git a/docs-mslearn/toolkit/hubs/upgrade.md b/docs-mslearn/toolkit/hubs/upgrade.md index e78f00ab2..eb7b112d6 100644 --- a/docs-mslearn/toolkit/hubs/upgrade.md +++ b/docs-mslearn/toolkit/hubs/upgrade.md @@ -3,7 +3,7 @@ title: Upgrade your FinOps hubs description: Learn how to upgrade your existing FinOps hub instance to the latest version, including necessary steps and considerations. author: flanakin ms.author: micflan -ms.date: 02/24/2026 +ms.date: 04/01/2026 ms.topic: how-to ms.service: finops ms.subservice: finops-toolkit diff --git a/docs-mslearn/toolkit/open-data.md b/docs-mslearn/toolkit/open-data.md index d5d3808dd..42b0fa0e1 100644 --- a/docs-mslearn/toolkit/open-data.md +++ b/docs-mslearn/toolkit/open-data.md @@ -4,7 +4,7 @@ description: Use open data to normalize and enhance your FinOps reporting, ensur ms.topic: concept-article author: flanakin ms.author: micflan -ms.date: 03/25/2026 +ms.date: 04/01/2026 ms.service: finops ms.subservice: finops-toolkit ms.reviewer: micflan diff --git a/docs-mslearn/toolkit/optimization-engine/configure-workspaces.md b/docs-mslearn/toolkit/optimization-engine/configure-workspaces.md index e54371904..d6d67ee78 100644 --- a/docs-mslearn/toolkit/optimization-engine/configure-workspaces.md +++ b/docs-mslearn/toolkit/optimization-engine/configure-workspaces.md @@ -3,7 +3,7 @@ title: Configure workspaces description: Include the VM performance logs available in your Log Analytics workspaces to get deeper insights and more accurate results. author: flanakin ms.author: micflan -ms.date: 02/24/2026 +ms.date: 04/01/2026 ms.topic: concept-article ms.service: finops ms.subservice: finops-toolkit diff --git a/docs-mslearn/toolkit/optimization-engine/customize.md b/docs-mslearn/toolkit/optimization-engine/customize.md index 5298abef8..c66be331b 100644 --- a/docs-mslearn/toolkit/optimization-engine/customize.md +++ b/docs-mslearn/toolkit/optimization-engine/customize.md @@ -3,7 +3,7 @@ title: Customize Azure optimization engine description: This article describes how to customize the Azure optimization engine settings according to your organization requirements. author: flanakin ms.author: micflan -ms.date: 02/24/2026 +ms.date: 04/01/2026 ms.topic: concept-article ms.service: finops ms.subservice: finops-toolkit diff --git a/docs-mslearn/toolkit/optimization-engine/faq.md b/docs-mslearn/toolkit/optimization-engine/faq.md index 4a0645be3..1c1dab208 100644 --- a/docs-mslearn/toolkit/optimization-engine/faq.md +++ b/docs-mslearn/toolkit/optimization-engine/faq.md @@ -3,7 +3,7 @@ title: Azure optimization engine FAQ description: This article covers frequently asked questions about the Azure Optimization Engine (AOE), including support, subscriptions, and currency. author: flanakin ms.author: micflan -ms.date: 02/24/2026 +ms.date: 04/01/2026 ms.topic: concept-article ms.service: finops ms.subservice: finops-toolkit diff --git a/docs-mslearn/toolkit/optimization-engine/overview.md b/docs-mslearn/toolkit/optimization-engine/overview.md index ca31872fc..7c7949bc0 100644 --- a/docs-mslearn/toolkit/optimization-engine/overview.md +++ b/docs-mslearn/toolkit/optimization-engine/overview.md @@ -3,7 +3,7 @@ title: Get started with the Azure Optimization Engine description: The Azure Optimization Engine (AOE) is an extensible solution designed to generate optimization recommendations for your Azure environment. author: flanakin ms.author: micflan -ms.date: 02/24/2026 +ms.date: 04/01/2026 ms.topic: how-to ms.service: finops ms.subservice: finops-toolkit diff --git a/docs-mslearn/toolkit/optimization-engine/reports.md b/docs-mslearn/toolkit/optimization-engine/reports.md index b64b8325e..5324f1c60 100644 --- a/docs-mslearn/toolkit/optimization-engine/reports.md +++ b/docs-mslearn/toolkit/optimization-engine/reports.md @@ -3,7 +3,7 @@ title: Azure optimization engine reports description: Visualize the Azure Optimization Engine's rich recommendations and insights through various reporting options, including Power BI reports. author: flanakin ms.author: micflan -ms.date: 02/24/2026 +ms.date: 04/01/2026 ms.topic: concept-article ms.service: finops ms.subservice: finops-toolkit diff --git a/docs-mslearn/toolkit/optimization-engine/setup-options.md b/docs-mslearn/toolkit/optimization-engine/setup-options.md index 23cc756c2..5c478a151 100644 --- a/docs-mslearn/toolkit/optimization-engine/setup-options.md +++ b/docs-mslearn/toolkit/optimization-engine/setup-options.md @@ -3,7 +3,7 @@ title: Setup options description: This article describes advance scenarios for setting up or upgrading Azure optimization engine (AOE). author: flanakin ms.author: micflan -ms.date: 02/24/2026 +ms.date: 04/01/2026 ms.topic: concept-article ms.service: finops ms.subservice: finops-toolkit diff --git a/docs-mslearn/toolkit/optimization-engine/suppress-recommendations.md b/docs-mslearn/toolkit/optimization-engine/suppress-recommendations.md index ad7478927..d5ec049d5 100644 --- a/docs-mslearn/toolkit/optimization-engine/suppress-recommendations.md +++ b/docs-mslearn/toolkit/optimization-engine/suppress-recommendations.md @@ -3,7 +3,7 @@ title: Suppress recommendations description: Learn how to adjust the Azure Optimization Engine recommendation results for your environment characteristics by suppressing irrelevant recommendations. author: flanakin ms.author: micflan -ms.date: 02/24/2026 +ms.date: 04/01/2026 ms.topic: concept-article ms.service: finops ms.subservice: finops-toolkit diff --git a/docs-mslearn/toolkit/optimization-engine/troubleshooting.md b/docs-mslearn/toolkit/optimization-engine/troubleshooting.md index 02b5b4585..ab6e2000a 100644 --- a/docs-mslearn/toolkit/optimization-engine/troubleshooting.md +++ b/docs-mslearn/toolkit/optimization-engine/troubleshooting.md @@ -3,7 +3,7 @@ title: Troubleshoot Azure Optimization Engine issues description: This article helps you troubleshoot common issues with Azure Optimization Engine deployment and runtime. author: flanakin ms.author: micflan -ms.date: 02/24/2026 +ms.date: 04/01/2026 ms.topic: troubleshooting ms.service: finops ms.subservice: finops-toolkit diff --git a/docs-mslearn/toolkit/power-bi/connector.md b/docs-mslearn/toolkit/power-bi/connector.md index dbb5f4dde..ef1163f76 100644 --- a/docs-mslearn/toolkit/power-bi/connector.md +++ b/docs-mslearn/toolkit/power-bi/connector.md @@ -3,7 +3,7 @@ title: Cost Management connector report description: Understand the Power BI report for the Cost Management connector, including cost overviews, commitment discounts, and savings insights. author: flanakin ms.author: micflan -ms.date: 02/24/2026 +ms.date: 04/01/2026 ms.topic: concept-article ms.service: finops ms.subservice: finops-toolkit diff --git a/docs-mslearn/toolkit/power-bi/cost-summary.md b/docs-mslearn/toolkit/power-bi/cost-summary.md index ba345dba3..ad650cee1 100644 --- a/docs-mslearn/toolkit/power-bi/cost-summary.md +++ b/docs-mslearn/toolkit/power-bi/cost-summary.md @@ -3,7 +3,7 @@ title: FinOps toolkit Cost summary report description: Learn about the Cost Summary Report in Power BI to identify top cost contributors, review cost changes over time, and summarize savings. author: flanakin ms.author: micflan -ms.date: 02/24/2026 +ms.date: 04/01/2026 ms.topic: concept-article ms.service: finops ms.subservice: finops-toolkit diff --git a/docs-mslearn/toolkit/power-bi/data-ingestion.md b/docs-mslearn/toolkit/power-bi/data-ingestion.md index 636bcdb77..efd25406a 100644 --- a/docs-mslearn/toolkit/power-bi/data-ingestion.md +++ b/docs-mslearn/toolkit/power-bi/data-ingestion.md @@ -3,7 +3,7 @@ title: Data ingestion report description: Learn about the Data Ingestion Report, which provides insights into the data ingested into your FinOps hub storage account. author: flanakin ms.author: micflan -ms.date: 02/24/2026 +ms.date: 04/01/2026 ms.topic: concept-article ms.service: finops ms.subservice: finops-toolkit diff --git a/docs-mslearn/toolkit/power-bi/governance.md b/docs-mslearn/toolkit/power-bi/governance.md index db446af03..a8f6d59ae 100644 --- a/docs-mslearn/toolkit/power-bi/governance.md +++ b/docs-mslearn/toolkit/power-bi/governance.md @@ -3,7 +3,7 @@ title: FinOps toolkit Governance report description: Summarize cloud governance posture including areas like compliance, security, operations, and resource management in Power BI. author: flanakin ms.author: micflan -ms.date: 02/24/2026 +ms.date: 04/01/2026 ms.topic: concept-article ms.service: finops ms.subservice: finops-toolkit diff --git a/docs-mslearn/toolkit/power-bi/help-me-choose.md b/docs-mslearn/toolkit/power-bi/help-me-choose.md index a0f1c3a38..06859ddfe 100644 --- a/docs-mslearn/toolkit/power-bi/help-me-choose.md +++ b/docs-mslearn/toolkit/power-bi/help-me-choose.md @@ -3,7 +3,7 @@ title: Choose a Power BI data source description: Learn about different ways to connect Power BI to your data to analyze and report on cloud costs, including connectors and exports. author: flanakin ms.author: micflan -ms.date: 02/24/2026 +ms.date: 04/01/2026 ms.topic: concept-article ms.service: finops ms.subservice: finops-toolkit diff --git a/docs-mslearn/toolkit/power-bi/invoicing.md b/docs-mslearn/toolkit/power-bi/invoicing.md index 7e33baf36..9ddde1e8b 100644 --- a/docs-mslearn/toolkit/power-bi/invoicing.md +++ b/docs-mslearn/toolkit/power-bi/invoicing.md @@ -3,7 +3,7 @@ title: FinOps toolkit Invoicing and chargeback report description: Learn about the Invoicing and chargeback report in Power BI to review and reconcile billed charges compared to your Microsoft Cloud invoice. author: flanakin ms.author: micflan -ms.date: 02/24/2026 +ms.date: 04/01/2026 ms.topic: concept-article ms.service: finops ms.subservice: finops-toolkit diff --git a/docs-mslearn/toolkit/power-bi/rate-optimization.md b/docs-mslearn/toolkit/power-bi/rate-optimization.md index d448f7c70..9990dcaf2 100644 --- a/docs-mslearn/toolkit/power-bi/rate-optimization.md +++ b/docs-mslearn/toolkit/power-bi/rate-optimization.md @@ -3,7 +3,7 @@ title: FinOps toolkit Rate optimization report description: Learn about the Rate Optimization Report in Power BI, which summarizes savings from commitment discounts like reservations and savings plans. author: flanakin ms.author: micflan -ms.date: 02/24/2026 +ms.date: 04/01/2026 ms.topic: concept-article ms.service: finops ms.subservice: finops-toolkit diff --git a/docs-mslearn/toolkit/power-bi/reports.md b/docs-mslearn/toolkit/power-bi/reports.md index 45ae4fdb0..c619c4c1c 100644 --- a/docs-mslearn/toolkit/power-bi/reports.md +++ b/docs-mslearn/toolkit/power-bi/reports.md @@ -3,7 +3,7 @@ title: FinOps toolkit Power BI reports description: Learn about the Power BI reports in the FinOps toolkit to customize and enhance your FinOps reporting and connect to Cost Management exports or FinOps hubs. author: flanakin ms.author: micflan -ms.date: 02/24/2026 +ms.date: 04/01/2026 ms.topic: how-to ms.service: finops ms.subservice: finops-toolkit diff --git a/docs-mslearn/toolkit/power-bi/setup.md b/docs-mslearn/toolkit/power-bi/setup.md index 093dabf24..6aeae6653 100644 --- a/docs-mslearn/toolkit/power-bi/setup.md +++ b/docs-mslearn/toolkit/power-bi/setup.md @@ -3,7 +3,7 @@ title: Set up Power BI reports description: Learn how to set up Power BI FinOps reports using the FinOps toolkit, customize visuals, and connect to your cost data for detailed analysis. author: flanakin ms.author: micflan -ms.date: 02/24/2026 +ms.date: 04/01/2026 ms.topic: how-to ms.service: finops ms.subservice: finops-toolkit diff --git a/docs-mslearn/toolkit/power-bi/template-app.md b/docs-mslearn/toolkit/power-bi/template-app.md index 6f4fc51dc..bfc33ae5f 100644 --- a/docs-mslearn/toolkit/power-bi/template-app.md +++ b/docs-mslearn/toolkit/power-bi/template-app.md @@ -3,7 +3,7 @@ title: Cost Management template app for Enterprise Agreement description: Learn about the Cost Management template app for Enterprise Agreement accounts, including its features, usage insights, and available reports. author: flanakin ms.author: micflan -ms.date: 02/24/2026 +ms.date: 04/01/2026 ms.topic: concept-article ms.service: finops ms.subservice: finops-toolkit diff --git a/docs-mslearn/toolkit/power-bi/workload-optimization.md b/docs-mslearn/toolkit/power-bi/workload-optimization.md index 228652c65..3624f4b6f 100644 --- a/docs-mslearn/toolkit/power-bi/workload-optimization.md +++ b/docs-mslearn/toolkit/power-bi/workload-optimization.md @@ -3,7 +3,7 @@ title: FinOps toolkit Workload optimization report description: Learn about the Workload optimization report, which identifies opportunities for rightsizing and removing unused resources to enhance efficiency. author: flanakin ms.author: micflan -ms.date: 02/24/2026 +ms.date: 04/01/2026 ms.topic: concept-article ms.service: finops ms.subservice: finops-toolkit diff --git a/docs-mslearn/toolkit/powershell/cost/add-finopsserviceprincipal.md b/docs-mslearn/toolkit/powershell/cost/add-finopsserviceprincipal.md index 19f05f4db..8d4a0262f 100644 --- a/docs-mslearn/toolkit/powershell/cost/add-finopsserviceprincipal.md +++ b/docs-mslearn/toolkit/powershell/cost/add-finopsserviceprincipal.md @@ -3,7 +3,7 @@ title: Add-FinOpsServicePrincipal command description: Grants the specified service principal or managed identity access to an Enterprise Agreement billing account or department. author: flanakin ms.author: micflan -ms.date: 02/24/2026 +ms.date: 04/01/2026 ms.topic: reference ms.service: finops ms.subservice: finops-toolkit diff --git a/docs-mslearn/toolkit/powershell/cost/cost-management-commands.md b/docs-mslearn/toolkit/powershell/cost/cost-management-commands.md index a606da5de..2542f4fbf 100644 --- a/docs-mslearn/toolkit/powershell/cost/cost-management-commands.md +++ b/docs-mslearn/toolkit/powershell/cost/cost-management-commands.md @@ -3,7 +3,7 @@ title: Cost Management commands description: Learn about PowerShell commands in the FinOpsToolkit module to support Cost Management capabilities. author: flanakin ms.author: micflan -ms.date: 02/24/2026 +ms.date: 04/01/2026 ms.topic: reference ms.service: finops ms.subservice: finops-toolkit diff --git a/docs-mslearn/toolkit/powershell/cost/get-finopscostexport.md b/docs-mslearn/toolkit/powershell/cost/get-finopscostexport.md index 8077fabf2..15efa2ac4 100644 --- a/docs-mslearn/toolkit/powershell/cost/get-finopscostexport.md +++ b/docs-mslearn/toolkit/powershell/cost/get-finopscostexport.md @@ -3,7 +3,7 @@ title: Get-FinOpsCostExport command description: Get a list of Cost Management exports for a given scope using the Get-FinOpsCostExport command in the FinOpsToolkit module. author: flanakin ms.author: micflan -ms.date: 02/24/2026 +ms.date: 04/01/2026 ms.topic: reference ms.service: finops ms.subservice: finops-toolkit diff --git a/docs-mslearn/toolkit/powershell/cost/new-finopscostexport.md b/docs-mslearn/toolkit/powershell/cost/new-finopscostexport.md index 06c7730dd..3d53ee583 100644 --- a/docs-mslearn/toolkit/powershell/cost/new-finopscostexport.md +++ b/docs-mslearn/toolkit/powershell/cost/new-finopscostexport.md @@ -3,7 +3,7 @@ title: New-FinOpsCostExport command description: Create a new Cost Management export for the specified scope using the New-FinOpsCostExport command in the FinOpsToolkit module. author: flanakin ms.author: micflan -ms.date: 02/24/2026 +ms.date: 04/01/2026 ms.topic: reference ms.service: finops ms.subservice: finops-toolkit diff --git a/docs-mslearn/toolkit/powershell/cost/remove-finopscostexport.md b/docs-mslearn/toolkit/powershell/cost/remove-finopscostexport.md index 9ad1cc949..6075309f4 100644 --- a/docs-mslearn/toolkit/powershell/cost/remove-finopscostexport.md +++ b/docs-mslearn/toolkit/powershell/cost/remove-finopscostexport.md @@ -3,7 +3,7 @@ title: Remove-FinOpsCostExport command description: Delete a Cost Management export and optionally data associated with the export using the Remove-FinOpsCostExport command in the FinOpsToolkit module. author: flanakin ms.author: micflan -ms.date: 02/24/2026 +ms.date: 04/01/2026 ms.topic: reference ms.service: finops ms.subservice: finops-toolkit diff --git a/docs-mslearn/toolkit/powershell/cost/start-finopscostexport.md b/docs-mslearn/toolkit/powershell/cost/start-finopscostexport.md index 031023211..1f04ced70 100644 --- a/docs-mslearn/toolkit/powershell/cost/start-finopscostexport.md +++ b/docs-mslearn/toolkit/powershell/cost/start-finopscostexport.md @@ -3,7 +3,7 @@ title: Start-FinOpsCostExport command description: Initiate a Cost Management export run for the most recent period using the Start-FinOpsCostExport command in the FinOpsToolkit module. author: flanakin ms.author: micflan -ms.date: 02/24/2026 +ms.date: 04/01/2026 ms.topic: reference ms.service: finops ms.subservice: finops-toolkit diff --git a/docs-mslearn/toolkit/powershell/data/get-finopspricingunit.md b/docs-mslearn/toolkit/powershell/data/get-finopspricingunit.md index 651dbdb8e..7e79f009a 100644 --- a/docs-mslearn/toolkit/powershell/data/get-finopspricingunit.md +++ b/docs-mslearn/toolkit/powershell/data/get-finopspricingunit.md @@ -3,7 +3,7 @@ title: Get-FinOpsPricingUnit command description: Get a pricing unit, distinct unit, and block size using the Get-FinOpsPricingUnit command in the FinOpsToolkit module. author: flanakin ms.author: micflan -ms.date: 02/24/2026 +ms.date: 04/01/2026 ms.topic: reference ms.service: finops ms.subservice: finops-toolkit diff --git a/docs-mslearn/toolkit/powershell/data/get-finopsregion.md b/docs-mslearn/toolkit/powershell/data/get-finopsregion.md index 46fc4cacf..ec651b1aa 100644 --- a/docs-mslearn/toolkit/powershell/data/get-finopsregion.md +++ b/docs-mslearn/toolkit/powershell/data/get-finopsregion.md @@ -3,7 +3,7 @@ title: Get-FinOpsRegion command description: Get an Azure region ID and name based on the specified resource location using the Get-FinOpsRegion command in the FinOpsToolkit module. author: flanakin ms.author: micflan -ms.date: 02/24/2026 +ms.date: 04/01/2026 ms.topic: reference ms.service: finops ms.subservice: finops-toolkit diff --git a/docs-mslearn/toolkit/powershell/data/get-finopsresourcetype.md b/docs-mslearn/toolkit/powershell/data/get-finopsresourcetype.md index b134e4518..6713f80d5 100644 --- a/docs-mslearn/toolkit/powershell/data/get-finopsresourcetype.md +++ b/docs-mslearn/toolkit/powershell/data/get-finopsresourcetype.md @@ -3,7 +3,7 @@ title: Get-FinOpsResourceType command description: Get an Azure resource type with readable display names, preview status, description, icon, and support links using the Get-FinOpsResourceType command. author: flanakin ms.author: micflan -ms.date: 02/24/2026 +ms.date: 04/01/2026 ms.topic: reference ms.service: finops ms.subservice: finops-toolkit diff --git a/docs-mslearn/toolkit/powershell/data/get-finopsservice.md b/docs-mslearn/toolkit/powershell/data/get-finopsservice.md index 549c81205..acf93cfc0 100644 --- a/docs-mslearn/toolkit/powershell/data/get-finopsservice.md +++ b/docs-mslearn/toolkit/powershell/data/get-finopsservice.md @@ -3,7 +3,7 @@ title: Get-FinOpsService command description: Get the name and category for a service, publisher, and cloud provider using the Get-FinOpsService command in the FinOpsToolkit module. author: flanakin ms.author: micflan -ms.date: 02/24/2026 +ms.date: 04/01/2026 ms.topic: reference ms.service: finops ms.subservice: finops-toolkit diff --git a/docs-mslearn/toolkit/powershell/data/open-data-commands.md b/docs-mslearn/toolkit/powershell/data/open-data-commands.md index 150d6e359..a18b4851d 100644 --- a/docs-mslearn/toolkit/powershell/data/open-data-commands.md +++ b/docs-mslearn/toolkit/powershell/data/open-data-commands.md @@ -3,7 +3,7 @@ title: Open data commands description: Learn about PowerShell commands available in the FinOpsToolkit module to work with FinOps open data and integrate datasets into your workflow. author: flanakin ms.author: micflan -ms.date: 02/24/2026 +ms.date: 04/01/2026 ms.topic: reference ms.service: finops ms.subservice: finops-toolkit diff --git a/docs-mslearn/toolkit/powershell/hubs/deploy-finopshub.md b/docs-mslearn/toolkit/powershell/hubs/deploy-finopshub.md index c1128f1e2..2a56c1b83 100644 --- a/docs-mslearn/toolkit/powershell/hubs/deploy-finopshub.md +++ b/docs-mslearn/toolkit/powershell/hubs/deploy-finopshub.md @@ -3,7 +3,7 @@ title: Deploy-FinOpsHub command description: Deploy a new or update an existing FinOps hub instance using the Deploy-FinOpsHub command in the FinOpsToolkit module. author: flanakin ms.author: micflan -ms.date: 02/24/2026 +ms.date: 04/01/2026 ms.topic: reference ms.service: finops ms.subservice: finops-toolkit diff --git a/docs-mslearn/toolkit/powershell/hubs/finops-hubs-commands.md b/docs-mslearn/toolkit/powershell/hubs/finops-hubs-commands.md index f37a9240b..70e6a8152 100644 --- a/docs-mslearn/toolkit/powershell/hubs/finops-hubs-commands.md +++ b/docs-mslearn/toolkit/powershell/hubs/finops-hubs-commands.md @@ -3,7 +3,7 @@ title: FinOps hubs automation description: Learn about PowerShell commands available in the FinOpsToolkit module that deploy and manage FinOps hubs. author: flanakin ms.author: micflan -ms.date: 02/24/2026 +ms.date: 04/01/2026 ms.topic: reference ms.service: finops ms.subservice: finops-toolkit diff --git a/docs-mslearn/toolkit/powershell/hubs/get-finopshub.md b/docs-mslearn/toolkit/powershell/hubs/get-finopshub.md index 19da32494..fcbb7cd0f 100644 --- a/docs-mslearn/toolkit/powershell/hubs/get-finopshub.md +++ b/docs-mslearn/toolkit/powershell/hubs/get-finopshub.md @@ -3,7 +3,7 @@ title: Get-FinOpsHub command description: Get details about a FinOps hub instance using the Get-FinOpsHub command in the FinOpsToolkit module. author: flanakin ms.author: micflan -ms.date: 02/24/2026 +ms.date: 04/01/2026 ms.topic: reference ms.service: finops ms.subservice: finops-toolkit diff --git a/docs-mslearn/toolkit/powershell/hubs/initialize-finopshubdeployment.md b/docs-mslearn/toolkit/powershell/hubs/initialize-finopshubdeployment.md index cc27dd772..b74e19c1c 100644 --- a/docs-mslearn/toolkit/powershell/hubs/initialize-finopshubdeployment.md +++ b/docs-mslearn/toolkit/powershell/hubs/initialize-finopshubdeployment.md @@ -3,7 +3,7 @@ title: Initialize-FinOpsHubDeployment command description: Initialize a FinOps hub deployment using the Initialize-FinOpsHubDeployment command in the FinOpsToolkit module. author: flanakin ms.author: micflan -ms.date: 02/24/2026 +ms.date: 04/01/2026 ms.topic: reference ms.service: finops ms.subservice: finops-toolkit diff --git a/docs-mslearn/toolkit/powershell/hubs/register-finopshubproviders.md b/docs-mslearn/toolkit/powershell/hubs/register-finopshubproviders.md index 904edcd66..295cce5ac 100644 --- a/docs-mslearn/toolkit/powershell/hubs/register-finopshubproviders.md +++ b/docs-mslearn/toolkit/powershell/hubs/register-finopshubproviders.md @@ -3,7 +3,7 @@ title: Register-FinOpsHubProviders command description: Register Azure resource providers required for FinOps hub using the Register-FinOpsHubProviders command in the FinOpsToolkit module. author: flanakin ms.author: micflan -ms.date: 02/24/2026 +ms.date: 04/01/2026 ms.topic: reference ms.service: finops ms.subservice: finops-toolkit diff --git a/docs-mslearn/toolkit/powershell/hubs/remove-finopshub.md b/docs-mslearn/toolkit/powershell/hubs/remove-finopshub.md index 64a8bda36..2eae99459 100644 --- a/docs-mslearn/toolkit/powershell/hubs/remove-finopshub.md +++ b/docs-mslearn/toolkit/powershell/hubs/remove-finopshub.md @@ -3,7 +3,7 @@ title: Remove-FinOpsHub command description: Remove a FinOps hub instance using the Remove-FinOpsHub command in the FinOpsToolkit module, with an option to keep the storage account hosting cost data. author: flanakin ms.author: micflan -ms.date: 02/24/2026 +ms.date: 04/01/2026 ms.topic: reference ms.service: finops ms.subservice: finops-toolkit diff --git a/docs-mslearn/toolkit/powershell/hubs/remove-finopshubscope.md b/docs-mslearn/toolkit/powershell/hubs/remove-finopshubscope.md index d216265d4..10821635d 100644 --- a/docs-mslearn/toolkit/powershell/hubs/remove-finopshubscope.md +++ b/docs-mslearn/toolkit/powershell/hubs/remove-finopshubscope.md @@ -3,7 +3,7 @@ title: Remove-FinOpsHubScope command description: Stops monitoring a scope within a FinOps hub instance and optionally remove the data using the Remove-FinOpsHubScope command in the FinOpsToolkit module. author: flanakin ms.author: micflan -ms.date: 02/24/2026 +ms.date: 04/01/2026 ms.topic: reference ms.service: finops ms.subservice: finops-toolkit diff --git a/docs-mslearn/toolkit/powershell/powershell-commands.md b/docs-mslearn/toolkit/powershell/powershell-commands.md index 073aff92d..d80414150 100644 --- a/docs-mslearn/toolkit/powershell/powershell-commands.md +++ b/docs-mslearn/toolkit/powershell/powershell-commands.md @@ -3,7 +3,7 @@ title: FinOps toolkit PowerShell module description: Automate and scale your FinOps efforts using the FinOps toolkit PowerShell module, which includes commands to manage FinOps solutions. author: flanakin ms.author: micflan -ms.date: 02/24/2026 +ms.date: 04/01/2026 ms.topic: reference ms.service: finops ms.subservice: finops-toolkit diff --git a/docs-mslearn/toolkit/powershell/toolkit/finops-toolkit-commands.md b/docs-mslearn/toolkit/powershell/toolkit/finops-toolkit-commands.md index b3ed3a6b6..d6f93a7c9 100644 --- a/docs-mslearn/toolkit/powershell/toolkit/finops-toolkit-commands.md +++ b/docs-mslearn/toolkit/powershell/toolkit/finops-toolkit-commands.md @@ -3,7 +3,7 @@ title: Toolkit commands description: This article summarizes PowerShell commands available for general FinOps toolkit operations, including getting details about toolkit releases. author: flanakin ms.author: micflan -ms.date: 02/24/2026 +ms.date: 04/01/2026 ms.topic: reference ms.service: finops ms.subservice: finops-toolkit diff --git a/docs-mslearn/toolkit/powershell/toolkit/get-finopstoolkitversion.md b/docs-mslearn/toolkit/powershell/toolkit/get-finopstoolkitversion.md index 7d8557ba6..977ad157e 100644 --- a/docs-mslearn/toolkit/powershell/toolkit/get-finopstoolkitversion.md +++ b/docs-mslearn/toolkit/powershell/toolkit/get-finopstoolkitversion.md @@ -3,7 +3,7 @@ title: Get-FinOpsToolkitVersion command description: Get available versions from published FinOps toolkit releases using the Get-FinOpsToolkitVersion command. author: flanakin ms.author: micflan -ms.date: 02/24/2026 +ms.date: 04/01/2026 ms.topic: reference ms.service: finops ms.subservice: finops-toolkit diff --git a/docs-mslearn/toolkit/roadmap.md b/docs-mslearn/toolkit/roadmap.md index 1a51448fb..cb5321a99 100644 --- a/docs-mslearn/toolkit/roadmap.md +++ b/docs-mslearn/toolkit/roadmap.md @@ -3,7 +3,7 @@ title: FinOps toolkit roadmap description: Explore the FinOps toolkit roadmap to learn about upcoming features, key themes, and initiatives planned for the future. author: flanakin ms.author: micflan -ms.date: 02/24/2026 +ms.date: 04/01/2026 ms.topic: reference ms.service: finops ms.subservice: finops-toolkit diff --git a/docs-mslearn/toolkit/workbooks/customize-workbooks.md b/docs-mslearn/toolkit/workbooks/customize-workbooks.md index 7ab6d18e5..8d4a0a8d6 100644 --- a/docs-mslearn/toolkit/workbooks/customize-workbooks.md +++ b/docs-mslearn/toolkit/workbooks/customize-workbooks.md @@ -3,7 +3,7 @@ title: Use and customize FinOps workbooks description: Learn how to install and customize FinOps workbooks to achieve FinOps goals, including cost recommendations, idle resource identification, and more. author: flanakin ms.author: micflan -ms.date: 02/24/2026 +ms.date: 04/01/2026 ms.topic: how-to ms.service: finops ms.subservice: finops-toolkit diff --git a/docs-mslearn/toolkit/workbooks/finops-workbooks-overview.md b/docs-mslearn/toolkit/workbooks/finops-workbooks-overview.md index 61e941f92..282abbc45 100644 --- a/docs-mslearn/toolkit/workbooks/finops-workbooks-overview.md +++ b/docs-mslearn/toolkit/workbooks/finops-workbooks-overview.md @@ -3,7 +3,7 @@ title: Deploy FinOps workbooks description: FinOps workbooks are Azure Monitor workbooks that help you implement FinOps capabilities, including optimization and governance, to achieve your FinOps goals. author: flanakin ms.author: micflan -ms.date: 02/24/2026 +ms.date: 04/01/2026 ms.topic: how-to ms.service: finops ms.subservice: finops-toolkit diff --git a/docs-mslearn/toolkit/workbooks/governance.md b/docs-mslearn/toolkit/workbooks/governance.md index fb1c6e69b..77d3b6666 100644 --- a/docs-mslearn/toolkit/workbooks/governance.md +++ b/docs-mslearn/toolkit/workbooks/governance.md @@ -3,7 +3,7 @@ title: Governance workbook description: Azure Monitor workbook focused on governance, providing an overview of your Azure environment's governance posture and compliance. author: flanakin ms.author: micflan -ms.date: 02/24/2026 +ms.date: 04/01/2026 ms.topic: concept-article ms.service: finops ms.subservice: finops-toolkit diff --git a/docs-mslearn/toolkit/workbooks/optimization.md b/docs-mslearn/toolkit/workbooks/optimization.md index 2504406fb..2ee0ee766 100644 --- a/docs-mslearn/toolkit/workbooks/optimization.md +++ b/docs-mslearn/toolkit/workbooks/optimization.md @@ -3,7 +3,7 @@ title: FinOps toolkit Optimization workbook description: The Azure Monitor workbook focuses on cost optimization, providing insights and recommendations for improving cost efficiency in your Azure environment. author: flanakin ms.author: micflan -ms.date: 02/24/2026 +ms.date: 04/01/2026 ms.topic: concept-article ms.service: finops ms.subservice: finops-toolkit diff --git a/src/powershell/Tests/Integration/HubsIngestionQueries.Tests.ps1 b/src/powershell/Tests/Integration/HubsIngestionQueries.Tests.ps1 new file mode 100644 index 000000000..60a8966e1 --- /dev/null +++ b/src/powershell/Tests/Integration/HubsIngestionQueries.Tests.ps1 @@ -0,0 +1,152 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +& "$PSScriptRoot/../Initialize-Tests.ps1" + +Describe 'HubsIngestionQueries' { + + BeforeDiscovery { + $repoRoot = (Resolve-Path "$PSScriptRoot/../../../..").Path + $queriesPath = Join-Path $repoRoot 'src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Recommendations/queries' + $queryFiles = @(Get-ChildItem -Path $queriesPath -Filter '*.json' -ErrorAction SilentlyContinue | ForEach-Object { + $json = Get-Content -Path $_.FullName -Raw | ConvertFrom-Json + @{ + Name = $_.Name + FullName = $_.FullName + BaseName = $_.BaseName + Query = $json.query + QueryEngine = $json.queryEngine + Dataset = $json.dataset + Type = $json.type + Source = $json.source + } + }) + } + + BeforeAll { + $context = Get-AzContext + if (-not $context) + { + throw 'Not authenticated to Azure. Run Connect-AzAccount first.' + } + + # Verify Az.ResourceGraph module is available + if (-not (Get-Module -ListAvailable -Name 'Az.ResourceGraph')) + { + throw 'Az.ResourceGraph module is not installed. Run Install-Module Az.ResourceGraph.' + } + Import-Module Az.ResourceGraph -ErrorAction Stop + } + + Context 'ARG query execution' { + + It 'Should execute without errors: ' -ForEach $queryFiles { + if ($QueryEngine -ne 'ResourceGraph') + { + Set-ItResult -Skipped -Because "query engine '$QueryEngine' is not ResourceGraph" + return + } + + Monitor "Executing $Name..." -Indent ' ' { + try + { + $results = Search-AzGraph -Query $Query -UseTenantScope -First 1 -ErrorAction Stop + Report "Returned $($results.Count) result(s)" + } + catch + { + Report "Query failed: $($_.Exception.Message)" -Exception $_ + throw + } + } + } + } + + Context 'ARG query result schema' { + + It 'Should return expected columns: ' -ForEach $queryFiles { + if ($QueryEngine -ne 'ResourceGraph') + { + Set-ItResult -Skipped -Because "query engine '$QueryEngine' is not ResourceGraph" + return + } + + Monitor "Validating result schema for $Name..." -Indent ' ' { + try + { + $results = Search-AzGraph -Query $Query -UseTenantScope -First 5 -ErrorAction Stop + } + catch + { + Set-ItResult -Inconclusive -Because "query execution failed: $($_.Exception.Message)" + return + } + + if ($results -and $results.Count -gt 0) + { + $firstResult = $results | Select-Object -First 1 + $columns = $firstResult.PSObject.Properties.Name + + # These are the columns ARG queries should output (x_Source* columns are added downstream by the ADF pipeline) + $expectedColumns = @( + 'ResourceId' + 'ResourceName' + 'SubAccountId' + 'SubAccountName' + 'x_RecommendationCategory' + 'x_RecommendationDate' + 'x_RecommendationDescription' + 'x_RecommendationDetails' + 'x_RecommendationId' + 'x_ResourceGroupName' + ) + + foreach ($col in $expectedColumns) + { + $columns | Should -Contain $col -Because "result should include column '$col'" + } + + # Verify no extra columns beyond what the schema expects + foreach ($col in $columns) + { + $expectedColumns | Should -Contain $col -Because "unexpected column '$col' found in query result; queries should only return columns mapped in the schema" + } + + Report "All $($expectedColumns.Count) expected columns present, no extra columns" + } + else + { + Report 'Query returned no results (column validation skipped)' + Set-ItResult -Inconclusive -Because 'no results returned to validate columns' + } + } + } + } + + Context 'ARG query performance' { + + It 'Should complete within 30 seconds: ' -ForEach $queryFiles { + if ($QueryEngine -ne 'ResourceGraph') + { + Set-ItResult -Skipped -Because "query engine '$QueryEngine' is not ResourceGraph" + return + } + + Monitor "Testing performance for $Name..." -Indent ' ' { + $stopwatch = [System.Diagnostics.Stopwatch]::StartNew() + try + { + $null = Search-AzGraph -Query $Query -UseTenantScope -First 1 -ErrorAction Stop + $stopwatch.Stop() + $ms = $stopwatch.ElapsedMilliseconds + Report "Completed in ${ms}ms" + $ms | Should -BeLessThan 30000 -Because 'ARG queries should complete within 30 seconds' + } + catch + { + Set-ItResult -Inconclusive -Because "query execution failed: $($_.Exception.Message)" + } + } + } + } +} diff --git a/src/powershell/Tests/Integration/Toolkit.Tests.ps1 b/src/powershell/Tests/Integration/Toolkit.Tests.ps1 index a183b3f06..86401243c 100644 --- a/src/powershell/Tests/Integration/Toolkit.Tests.ps1 +++ b/src/powershell/Tests/Integration/Toolkit.Tests.ps1 @@ -11,7 +11,8 @@ Describe 'Get-FinOpsToolkitVersion' { # Helper function to normalize version strings for [version] parsing # Single-part versions like "12" need to become "12.0" for [version] to parse them - function NormalizeVersion($ver) { + function NormalizeVersion($ver) + { if ($ver -notmatch '\.') { return "$ver.0" } return $ver } @@ -54,7 +55,7 @@ Describe 'Get-FinOpsToolkitVersion' { CheckFile "optimization-workbook-v$verStr.zip" $null '0.5' # Power BI - CheckFile "FinOpsToolkitData.pbix" '12.0' $null + CheckFile "FinOpsToolkitData.pbix" '12.0' '12.0' # Accidentally added in v12 CheckFile "PowerBI-demo.zip" '0.7' $null CheckFile "PowerBI-kql.zip" '0.7' $null CheckFile "PowerBI-storage.zip" '0.7' $null @@ -80,6 +81,10 @@ Describe 'Get-FinOpsToolkitVersion' { CheckFile "ResourceTypes.json" '0.2' $null CheckFile "Services.csv" '0.1' $null + # Meeting invites + CheckFile "contributor-sync.ics" '13.0' $null + CheckFile "office-hours.ics" '13.0' $null + # Deprecated / renamed CheckFile "CommitmentDiscounts.pbit" '0.2' '0.3' CheckFile "CommitmentDiscounts.pbix" $null '0.3' diff --git a/src/powershell/Tests/Unit/HubsIngestionQueries.Tests.ps1 b/src/powershell/Tests/Unit/HubsIngestionQueries.Tests.ps1 new file mode 100644 index 000000000..f3aa5d8b0 --- /dev/null +++ b/src/powershell/Tests/Unit/HubsIngestionQueries.Tests.ps1 @@ -0,0 +1,146 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +Describe 'HubsIngestionQueries' { + + BeforeDiscovery { + $repoRoot = (Resolve-Path "$PSScriptRoot/../../../..").Path + $queriesPath = Join-Path $repoRoot 'src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Recommendations/queries' + $schemasPath = Join-Path $repoRoot 'src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Recommendations/schemas' + $schemaFileNames = @(Get-ChildItem -Path $schemasPath -Filter '*.json' -ErrorAction SilentlyContinue | ForEach-Object { $_.Name }) + # Convert FileInfo to hashtables so Pester -ForEach iterates correctly + $queryFiles = @(Get-ChildItem -Path $queriesPath -Filter '*.json' -ErrorAction SilentlyContinue | ForEach-Object { + @{ Name = $_.Name; FullName = $_.FullName; BaseName = $_.BaseName; SchemaFileNames = $schemaFileNames } + }) + $schemaFiles = @(Get-ChildItem -Path $schemasPath -Filter '*.json' -ErrorAction SilentlyContinue | ForEach-Object { + @{ Name = $_.Name; FullName = $_.FullName; BaseName = $_.BaseName } + }) + } + + BeforeAll { + $repoRoot = (Resolve-Path "$PSScriptRoot/../../../..").Path + $queriesPath = Join-Path $repoRoot 'src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Recommendations/queries' + $schemasPath = Join-Path $repoRoot 'src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Recommendations/schemas' + $queryFileCount = @(Get-ChildItem -Path $queriesPath -Filter '*.json' -ErrorAction SilentlyContinue).Count + $schemaFileCount = @(Get-ChildItem -Path $schemasPath -Filter '*.json' -ErrorAction SilentlyContinue).Count + $knownEngines = @('ResourceGraph') + $requiredQueryFields = @('dataset', 'provider', 'query', 'queryEngine', 'scope', 'source', 'type', 'version') + + # Derive known groups from Recommendations/app.bicep parameters. + # Non-core groups need a corresponding "enable{Group}Recommendations" bool parameter in app.bicep. + $appBicepPath = Join-Path $repoRoot 'src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Recommendations/app.bicep' + $appBicepContent = Get-Content -Path $appBicepPath -Raw + $knownGroups = @('core') + @([regex]::Matches($appBicepContent, 'param enable(\w+)Recommendations bool') | ForEach-Object { $_.Groups[1].Value.ToLower() }) + } + + Context 'Query files' { + + It 'Should have at least one query file' { + $queryFileCount | Should -BeGreaterThan 0 + } + + It 'Should be valid JSON: ' -ForEach $queryFiles { + { Get-Content -Path $FullName -Raw | ConvertFrom-Json } | Should -Not -Throw + } + + It 'Should have all required fields: ' -ForEach $queryFiles { + $json = Get-Content -Path $FullName -Raw | ConvertFrom-Json + foreach ($field in $requiredQueryFields) + { + $json.PSObject.Properties.Name | Should -Contain $field -Because "query file '$Name' is missing required field '$field'" + } + } + + It 'Should not have empty fields: ' -ForEach $queryFiles { + $json = Get-Content -Path $FullName -Raw | ConvertFrom-Json + foreach ($field in $requiredQueryFields) + { + $json.$field | Should -Not -BeNullOrEmpty -Because "field '$field' in '$Name' should not be empty" + } + } + + It 'Should use a known query engine: ' -ForEach $queryFiles { + $json = Get-Content -Path $FullName -Raw | ConvertFrom-Json + $json.queryEngine | Should -BeIn $knownEngines -Because "queryEngine '$($json.queryEngine)' in '$Name' is not a known engine ($($knownEngines -join ', '))" + } + + It 'Should match naming convention: ' -ForEach $queryFiles { + $Name | Should -Match '^[A-Za-z]+-[A-Za-z]+-[A-Za-z0-9]+\.json$' -Because "query file '$Name' should follow the '{Dataset}-{Provider}-{Name}.json' naming convention" + } + + It 'Should use a known query group: ' -ForEach $queryFiles { + $json = Get-Content -Path $FullName -Raw | ConvertFrom-Json + $group = if ($json.PSObject.Properties['group'] -and $json.group) { $json.group } else { 'core' } + $group | Should -BeIn $knownGroups -Because "query group '$group' in '$Name' is not a known group ($($knownGroups -join ', ')). Add the group to Build-HubIngestionQueries.ps1 `$groupConfig` and Recommendations/app.bicep before using it." + } + + It 'Should be consistent with dataset field: ' -ForEach $queryFiles { + $json = Get-Content -Path $FullName -Raw | ConvertFrom-Json + $expectedPrefix = "$($json.dataset)-" + $BaseName | Should -BeLike "$expectedPrefix*" -Because "file name '$Name' should start with dataset '$($json.dataset)-'" + } + } + + Context 'Schema files' { + + It 'Should have at least one schema file' { + $schemaFileCount | Should -BeGreaterThan 0 + } + + It 'Should be valid JSON: ' -ForEach $schemaFiles { + { Get-Content -Path $FullName -Raw | ConvertFrom-Json } | Should -Not -Throw + } + + It 'Should have a translator property: ' -ForEach $schemaFiles { + $json = Get-Content -Path $FullName -Raw | ConvertFrom-Json + $json.PSObject.Properties.Name | Should -Contain 'translator' + } + + It 'Should have translator mappings: ' -ForEach $schemaFiles { + $json = Get-Content -Path $FullName -Raw | ConvertFrom-Json + $json.translator.PSObject.Properties.Name | Should -Contain 'mappings' + $json.translator.mappings.Count | Should -BeGreaterThan 0 + } + + It 'Should be TabularTranslator type: ' -ForEach $schemaFiles { + $json = Get-Content -Path $FullName -Raw | ConvertFrom-Json + $json.translator.type | Should -Be 'TabularTranslator' + } + + It 'Should have source and sink in mappings: ' -ForEach $schemaFiles { + $json = Get-Content -Path $FullName -Raw | ConvertFrom-Json + foreach ($mapping in $json.translator.mappings) + { + $mapping.PSObject.Properties.Name | Should -Contain 'source' -Because 'each mapping needs a source' + $mapping.PSObject.Properties.Name | Should -Contain 'sink' -Because 'each mapping needs a sink' + $mapping.source.path | Should -Not -BeNullOrEmpty -Because 'source path should not be empty' + $mapping.sink.path | Should -Not -BeNullOrEmpty -Because 'sink path should not be empty' + } + } + } + + Context 'Query-to-schema consistency' { + + It 'Should have a matching schema file: ' -ForEach $queryFiles { + $json = Get-Content -Path $FullName -Raw | ConvertFrom-Json + $expectedSchemaName = "$($json.dataset.ToLower())_$($json.version).json" + $SchemaFileNames | Should -Contain $expectedSchemaName -Because "query '$Name' references dataset '$($json.dataset)' version '$($json.version)' but no schema file '$expectedSchemaName' exists" + } + } + + Context 'Bicep compilation' { + + It 'finops-hub template should compile without errors' { + $mainBicep = Join-Path $repoRoot 'src/templates/finops-hub/main.bicep' + if (Get-Command 'bicep' -ErrorAction SilentlyContinue) + { + $result = bicep build $mainBicep --stdout 2>&1 + $LASTEXITCODE | Should -Be 0 -Because "Bicep compilation failed: $($result | Out-String)" + } + else + { + Set-ItResult -Skipped -Because 'bicep CLI not found' + } + } + } +} diff --git a/src/scripts/Build-HubIngestionQueries.ps1 b/src/scripts/Build-HubIngestionQueries.ps1 new file mode 100644 index 000000000..89738a2b5 --- /dev/null +++ b/src/scripts/Build-HubIngestionQueries.ps1 @@ -0,0 +1,172 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +<# + .SYNOPSIS + Generates Bicep loadTextContent entries for ingestion query files. + + .DESCRIPTION + Scans query JSON files in the Recommendations app and generates the corresponding + Bicep variable blocks in app.bicep. Each query file specifies an opt-in group via + an optional "group" field. Files without a group are added to the core set. + + This script runs as a post-copy build step, modifying the release copy of app.bicep + rather than the source files. The source app.bicep contains placeholder markers that + are replaced with the generated content during the build. + + .PARAMETER DestDir + Required. Path to the finops-hub template destination (release) directory. + + .EXAMPLE + ./Build-HubIngestionQueries.ps1 -DestDir ./release/finops-hub + + Regenerates the loadTextContent entries in the release copy of Recommendations/app.bicep. + + .LINK + https://github.com/microsoft/finops-toolkit/blob/dev/src/scripts/README.md +#> + +[CmdletBinding()] +param( + [Parameter(Mandatory)][string]$DestDir +) + +$queriesPath = Join-Path $DestDir 'modules/Microsoft.FinOpsHubs/Recommendations/queries' +$appBicepPath = Join-Path $DestDir 'modules/Microsoft.FinOpsHubs/Recommendations/app.bicep' + +if (-not (Test-Path $queriesPath)) +{ + Write-Verbose "No queries directory found at $queriesPath; skipping" + return +} + +if (-not (Test-Path $appBicepPath)) +{ + Write-Warning "Recommendations app.bicep not found at $appBicepPath; skipping" + return +} + +# Read all query files and group them +$queryFiles = Get-ChildItem -Path $queriesPath -Filter '*.json' | Sort-Object Name +$groups = @{} + +foreach ($file in $queryFiles) +{ + $json = Get-Content -Path $file.FullName -Raw | ConvertFrom-Json + $group = if ($json.PSObject.Properties['group'] -and $json.group) { $json.group } else { 'core' } + if (-not $groups.ContainsKey($group)) + { + $groups[$group] = @() + } + $groups[$group] += $file +} + +Write-Verbose "Found $($queryFiles.Count) query file(s) in $($groups.Count) group(s)" + +# Generate the Bicep variable block for each group +function Format-BicepVar($varName, $files, $conditional) +{ + $lines = @() + if ($conditional) + { + $lines += "var $varName = $conditional {" + } + else + { + $lines += "var $varName = {" + } + foreach ($file in $files) + { + $key = $file.BaseName + $lines += " '$key': loadTextContent('queries/$($file.Name)')" + } + if ($conditional) + { + $lines += '} : {}' + } + else + { + $lines += '}' + } + return $lines -join "`n" +} + +# Known group-to-variable mappings with their conditional expressions +$groupConfig = @{ + 'core' = @{ VarName = 'coreQueryFiles'; Conditional = $null } + 'ahb' = @{ VarName = 'ahbQueryFiles'; Conditional = 'enableAHBRecommendations ?' } + 'spot' = @{ VarName = 'spotQueryFiles'; Conditional = 'enableSpotRecommendations ?' } +} + +# Build the generated block +$startMarker = '// ' +$endMarker = '// ' + +$generatedLines = @($startMarker) +$varNames = @() + +foreach ($groupName in @('core', 'ahb', 'spot')) +{ + if (-not $groups.ContainsKey($groupName)) { continue } + + $config = $groupConfig[$groupName] + $varNames += $config.VarName + + if ($generatedLines.Count -gt 1) { $generatedLines += '' } + + # Add comment for non-core groups + switch ($groupName) + { + 'core' { $generatedLines += '// Load query files -- core recommendations are always included' } + 'ahb' { $generatedLines += '// Optional: Azure Hybrid Benefit recommendations (may generate noise without on-premises licenses)' } + 'spot' { $generatedLines += '// Optional: Spot VM recommendations (may generate noise for non-interruptible workloads)' } + } + + $generatedLines += Format-BicepVar $config.VarName $groups[$groupName] $config.Conditional +} + +# Handle any unknown groups +foreach ($groupName in ($groups.Keys | Sort-Object)) +{ + if ($groupConfig.ContainsKey($groupName)) { continue } + + Write-Warning "Unknown query group '$groupName' found; adding as opt-in variable" + $varName = "${groupName}QueryFiles" + $paramName = "enable$($groupName.Substring(0,1).ToUpper())$($groupName.Substring(1))Recommendations" + $varNames += $varName + + $generatedLines += '' + $generatedLines += "// Optional: $groupName recommendations" + $generatedLines += Format-BicepVar $varName $groups[$groupName] "$paramName ?" +} + +# Add the union line +$generatedLines += '' +$generatedLines += "var queryFiles = union($($varNames -join ', '))" +$generatedLines += $endMarker + +$generatedBlock = $generatedLines -join "`n" + +# Read existing app.bicep and replace the generated section +$bicepContent = Get-Content -Path $appBicepPath -Raw + +# Match from start marker through end marker +$pattern = "(?ms)$([regex]::Escape($startMarker)).*?$([regex]::Escape($endMarker))" +if ($bicepContent -match $pattern) +{ + $newContent = $bicepContent -replace $pattern, $generatedBlock + if ($newContent -ne $bicepContent) + { + $newContent | Out-File -FilePath $appBicepPath -Encoding utf8 -NoNewline + Write-Host " Updated $($queryFiles.Count) query entries in app.bicep" + } + else + { + Write-Verbose " app.bicep is already up to date" + } +} +else +{ + Write-Warning "Could not find generated section markers in app.bicep; manual update required" + Write-Warning "Expected markers: $startMarker ... $endMarker" +} diff --git a/src/scripts/Build-QueryCatalog.ps1 b/src/scripts/Build-QueryCatalog.ps1 new file mode 100644 index 000000000..bb23eccf8 --- /dev/null +++ b/src/scripts/Build-QueryCatalog.ps1 @@ -0,0 +1,96 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +<# + .SYNOPSIS + Generates the query-catalog.md knowledge file from KQL source files. + + .DESCRIPTION + Reads all .kql files from src/queries/catalog, includes the full file content + (comments and query), replaces Costs() with Costs_v1_2(), and writes a combined + markdown file to the Copilot Studio knowledge folder. + + .EXAMPLE + ./Build-QueryCatalog + + Generates query-catalog.md from all KQL files in the catalog. + + .LINK + https://github.com/microsoft/finops-toolkit/blob/dev/src/scripts/README.md +#> + +[CmdletBinding()] +param() + +$ErrorActionPreference = 'Stop' + +$queriesDir = "$PSScriptRoot/../queries/catalog" +$outputFile = "$PSScriptRoot/../templates/finops-hub-copilot-studio/knowledge/query-catalog.md" + +# Queries that require cross-database references or special setup +$excludeFiles = @('costs-enriched-base') + +if (-not (Test-Path $queriesDir)) +{ + Write-Error "Queries directory not found: $queriesDir" + return +} + +Write-Host "Building query catalog..." + +# Start with the header +$output = @" + + +# FinOps Hub KQL query catalog + +This document contains ready-to-use KQL query patterns for common FinOps analysis tasks. Use these as templates when answering cost questions. Always execute queries via the Kusto MCP tool — never answer from this document alone. + +All queries use the ``Costs_v1_2()`` function unless noted otherwise. +"@ + +# Process each KQL file +$kqlFiles = Get-ChildItem "$queriesDir/*.kql" -ErrorAction SilentlyContinue ` + | Where-Object { $excludeFiles -notcontains $_.BaseName } ` + | Sort-Object Name +if (-not $kqlFiles -or $kqlFiles.Count -eq 0) +{ + Write-Warning "No .kql files found in $queriesDir" + return +} +Write-Verbose "Found $($kqlFiles.Count) KQL file(s) to process" + +foreach ($file in $kqlFiles) +{ + Write-Verbose " Processing: $($file.Name)" + $content = (Get-Content $file.FullName -Raw).TrimEnd() + + # Extract the title from the filename (e.g., "monthly-cost-trend" -> "Monthly cost trend") + $title = ($file.BaseName -replace '-', ' ') + $title = $title.Substring(0, 1).ToUpper() + $title.Substring(1) + + # Replace Costs() with Costs_v1_2() + $content = $content -replace '\bCosts\(\)', 'Costs_v1_2()' + + # Guard against triple backticks in KQL content breaking the markdown code fence + if ($content -match '``````') + { + Write-Warning "Skipping $($file.Name): contains triple backticks that would break markdown" + continue + } + + # Build the section + $output += "`n`n## $title`n`n``````kusto`n$content`n``````" +} + +# Write output +$outputDir = Split-Path $outputFile -Parent +if (-not (Test-Path $outputDir)) +{ + New-Item -ItemType Directory -Path $outputDir -Force | Out-Null +} + +$output += "`n" +$output | Out-File $outputFile -Encoding utf8 -NoNewline +Write-Host " Generated: $outputFile" +Write-Host "" diff --git a/src/scripts/Build-Toolkit.ps1 b/src/scripts/Build-Toolkit.ps1 index 44daa0847..f9ec6848b 100644 --- a/src/scripts/Build-Toolkit.ps1 +++ b/src/scripts/Build-Toolkit.ps1 @@ -143,6 +143,14 @@ else Write-Verbose "No workbook templates found matching pattern: $Template" } +# Generate query catalog +if ($Template -eq "*" -or $Template -eq "finops-hub-copilot-studio") +{ + Write-Host "Building query catalog..." + Write-Verbose "Generating query-catalog.md from KQL source files" + & "$PSScriptRoot/Build-QueryCatalog.ps1" +} + # Package templates Write-Verbose "Searching for templates to package..." $templates = Get-ChildItem -Path "$PSScriptRoot/../templates/*", "$PSScriptRoot/../optimization-engine*" -Directory -ErrorAction SilentlyContinue @@ -175,7 +183,7 @@ $templates | ForEach-Object { $buildConfig = Get-Content "$_/.build.config" -ErrorAction SilentlyContinue | ConvertFrom-Json -Depth 10 # Backfill config options to avoid null references - (@{ ignore = @(); combineKql = @{}; rename = @{}; variableExpansion = @() }).PSObject.Properties | ForEach-Object { + (@{ ignore = @(); combineKql = @{}; rename = @{}; variableExpansion = @(); scripts = @() }).PSObject.Properties | ForEach-Object { if (-not $buildConfig.PSObject.Properties[$_.Name]) { $buildConfig | Add-Member -MemberType NoteProperty -Name $_.Name -Value $_.Value @@ -194,6 +202,30 @@ $templates | ForEach-Object { Write-Verbose " Copying $($sourceFiles.Count) items from source to destination" $sourceFiles | Copy-Item -Destination $destDir -Recurse + # Run custom build scripts after copying files (operates on release copy, not source) + if ($buildConfig.scripts.Count -gt 0) + { + Write-Host " Running custom build scripts..." + Write-Verbose " Processing $($buildConfig.scripts.Count) custom script(s)" + $buildConfig.scripts | ForEach-Object { + $scriptPath = "$PSScriptRoot/$_" + if (Test-Path $scriptPath) + { + Write-Verbose " Running: $_" + $LASTEXITCODE = 0 + & $scriptPath -DestDir $destDir + if (-not $? -or ($LASTEXITCODE -and $LASTEXITCODE -ne 0)) + { + throw "Custom build script '$_' failed" + } + } + else + { + throw "Custom build script not found: $scriptPath" + } + } + } + # Remove ignored files $ignoredFiles = (Get-Content "$srcDir/.buildignore" -ErrorAction SilentlyContinue) + $buildConfig.ignore if ($ignoredFiles.Length) diff --git a/src/scripts/Deploy-Hub.ps1 b/src/scripts/Deploy-Hub.ps1 index 900e88258..b48779470 100644 --- a/src/scripts/Deploy-Hub.ps1 +++ b/src/scripts/Deploy-Hub.ps1 @@ -59,7 +59,7 @@ Optional. PR number for CI deployments. Resources are named "pr-{number}" or "pr-{number}-{name}" when -Name is also specified. .PARAMETER HubName - Optional. Name of the hub instance. Default: "hub", or "{pr}{name}{initials}" when -PR or -Name is specified. + Optional. Name of the hub instance. Default: "hub", or "{name}{initials}" when -PR or -Name is specified. .PARAMETER ADX Optional. Name of the Azure Data Explorer cluster. Overrides the "{initials}-{name}" convention. Only used when not using -StorageOnly or -Fabric. @@ -228,12 +228,12 @@ if ($ManagedExports -and -not $Scope) # Build parameters $params = @{} -# Hub name — use "{pr}{name}{initials}" when -PR or -Name is specified, otherwise "hub" +# Hub name — use "{name}{initials}" when -PR or -Name is specified, otherwise "hub" if ($HubName) { $params.hubName = $HubName } elseif ($PR -or $PSBoundParameters.ContainsKey('Name')) { $explicitName = if ($PSBoundParameters.ContainsKey('Name')) { $Name } else { $null } - $hubParts = @($PR, $explicitName, $initials) | Where-Object { $_ } + $hubParts = @($explicitName, $initials) | Where-Object { $_ } $params.hubName = (($hubParts -join '') -replace '[^a-zA-Z0-9]', '').ToLower() } else { $params.hubName = "hub" } diff --git a/src/scripts/README.md b/src/scripts/README.md index dc231758d..6aeb2c5ad 100644 --- a/src/scripts/README.md +++ b/src/scripts/README.md @@ -188,11 +188,12 @@ All resources use an `{initials}-{name}` naming convention where initials are pu | `‑ResourceGroup` | Optional. Name of the resource group. Overrides the `{initials}-{name}` convention. | | `‑Fabric` | Optional. Deploy with Microsoft Fabric. Provide the eventhouse query URI. | | `‑StorageOnly` | Optional. Deploy a storage-only hub (no Azure Data Explorer or Fabric). | -| `‑Remove` | Optional. Remove test environments. With a name, deletes the target RG. Alone, lists all `{initials}-*`. | -| `‑PR` | Optional. PR number for CI deployments. Resources are named `pr-{number}` or `pr-{number}-{name}` when `-Name` is also specified. | +| `‑Recommendations` | Optional. Enable recommendations with all noisy recommendation types (AHB, Spot). | | `‑Scope` | Optional. Azure scope ID for cost data exports (e.g., `/subscriptions/{id}`). With `-ManagedExports`, enables managed exports. Without it, creates exports manually. | | `‑ManagedExports` | Optional. Use managed exports instead of manual exports. Requires `-Scope`. Passes `scopesToMonitor` to the template and grants the hub identity required roles. | +| `‑PR` | Optional. PR number for CI deployments. Resources are named `pr-{number}` or `pr-{number}-{name}` when `-Name` is also specified. | | `‑Location` | Optional. Azure location. Default: `westus`. | +| `‑Remove` | Optional. Remove test environments. With a name, deletes the target RG. Alone, lists all `{initials}-*`. | | `‑Build` | Optional. Build the template before deploying. | | `‑WhatIf` | Optional. Validate the deployment without making changes. | diff --git a/src/scripts/Test-PowerShell.ps1 b/src/scripts/Test-PowerShell.ps1 index 9d2ceba1d..fe2b6c5aa 100644 --- a/src/scripts/Test-PowerShell.ps1 +++ b/src/scripts/Test-PowerShell.ps1 @@ -129,7 +129,7 @@ else if ($Data) { $testsToRun += '*-OpenData*', '*-FinOpsPricingUnit*', '*-FinOpsRegion*', '*-FinOpsResourceType*', '*-FinOpsService*' } if ($Exports) { $testsToRun += '*-FinOpsCostExport*', 'CostExports.Tests.ps1' } if ($FOCUS) { $testsToRun += '*-FinOpsSchema*', 'FOCUS.Tests.ps1' } - if ($Hubs) { $testsToRun += '*-FinOpsHub*', '*-Hub*', 'Hubs.Tests.ps1' } + if ($Hubs) { $testsToRun += '*-FinOpsHub*', '*-Hub*', 'Hubs*.Tests.ps1' } if ($Toolkit) { $testsToRun += 'Toolkit.Tests.ps1', '*-FinOpsToolkit*' } if ($Workbooks) { $testsToRun += '*Workbook*' } if ($Actions) { $testsToRun += 'Action.*.Tests.ps1' } diff --git a/src/templates/finops-hub-copilot-studio/.build.config b/src/templates/finops-hub-copilot-studio/.build.config new file mode 100644 index 000000000..d48d65e3f --- /dev/null +++ b/src/templates/finops-hub-copilot-studio/.build.config @@ -0,0 +1,3 @@ +{ + "unversionedZip": true +} diff --git a/src/templates/finops-hub-copilot-studio/agent-instructions.md b/src/templates/finops-hub-copilot-studio/agent-instructions.md new file mode 100644 index 000000000..49102acc8 --- /dev/null +++ b/src/templates/finops-hub-copilot-studio/agent-instructions.md @@ -0,0 +1,128 @@ +# FinOps Hub Agent + +You are a FinOps analyst. Answer cost questions by running KQL against FinOps Hub. + +## Environment + +``` +Cluster URI: .kusto.windows.net +Database: Hub +``` + +## Knowledge references (use to BUILD queries, not to answer directly) + +- **`schema-reference.md`** — Column names, types, usage notes, and edge cases for `Costs_v1_2()`. Check BEFORE every query. Never quote as answers. +- **`query-catalog.md`** — Ready-to-use KQL query templates for cost breakdowns, trends, anomalies, savings, forecasting, and commitment analysis. Adapt and execute; never return as-is. +- **`weekly-report-guide.md`** — Workflow for structured weekly cost anomaly reports with severity classification and report structure. Follow when asked for weekly report. + +## First interaction — setup + +On first interaction, run these steps automatically: + +**Step 1: Detect currency** + +```kusto +Costs_v1_2() | where ChargePeriodStart >= ago(7d) | summarize count() by BillingCurrency | top 1 by count_ +``` + +Use the returned currency symbol for the session ($ for USD, CA$ for CAD, € for EUR, £ for GBP, etc.). + +**Step 2: Scope selection** +Query and present available scopes. Ask the user to pick ONE: + +| Scope | Discovery query | Filter for all queries | +| --------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------- | +| Billing account | `... \| distinct BillingAccountName` | `BillingAccountName == ""` | +| Subscription | `... \| distinct SubAccountName` | `SubAccountName == ""` | +| Resource group | `... \| distinct x_ResourceGroupName` | `x_ResourceGroupName == ""` | +| Tag | First query tag keys: `... \| mv-expand bagexpansion=array Tags \| summarize count() by tostring(Tags[0]) \| top 10 by count_` Then query values for chosen key. | `Tags[''] == ""` | +| All | No query needed | No filter | + +All discovery queries use: `Costs_v1_2() | where ChargePeriodStart >= ago(7d)` + +Apply the selected scope filter to EVERY subsequent query. Mention active scope in each response. User can change scope anytime. + +## Tools + +- **Kusto Query MCP Server** — execute KQL queries against the Hub database. Primary tool for all cost questions. +- **Microsoft Learn Docs MCP** — look up FinOps concepts, FOCUS spec, Azure service details. Use for domain knowledge, not cost data. + +## Core rules + +1. ALWAYS query `Costs_v1_2()` for cost data. NEVER query raw tables directly. +2. VERIFY column names against `schema-reference.md` before every query. +3. SHOW the KQL query you will run before executing it. +4. NEVER guess column names or values. If unsure, check `schema-reference.md`. +5. NEVER answer cost questions from reference files alone. Always execute a query. +6. State your confidence level (high/medium/low) with every answer. +7. Round costs to 2 decimal places. +8. Never mix marketplace (x_PublisherCategory == "Vendor") with cloud provider costs in the same analysis. Always report marketplace separately. + +## Cost metrics + +- **EffectiveCost** = default metric. Amortized cost after all discounts. Use unless asked otherwise. +- **BilledCost** = invoice/cash-flow amount. Use for "what did we pay" or AP questions. +- **ListCost** = full retail price. Use as baseline for savings calculations. +- **ContractedCost** = negotiated rate before commitment discounts. +- Savings = `ListCost - EffectiveCost` (total) or `ContractedCost - EffectiveCost` (commitment only). + +## Time filtering + +- Filter on `ChargePeriodStart` (datetime, daily granularity). +- For period ends (`ChargePeriodEnd`, `BillingPeriodEnd`), use `<` not `<=` (exclusive). +- `BillingPeriodStart` is always the 1st of the month. Use for calendar-month grouping. +- Default to last 30 days if no time range specified. +- **Data lag:** Azure cost data has 24-48h lag. Use `startofday(ago(3d))` as analysis end date for daily/weekly queries to avoid incomplete data. + +## Key dimensions + +- **Org:** `BillingAccountName` > `SubAccountName` (subscription) > `x_ResourceGroupName` > `ResourceName` (display only; group on `ResourceId`). Also: `x_InvoiceSectionName`, `x_AccountName`, `Tags`. +- **Service:** `ServiceCategory` > `ServiceName` > `x_SkuMeterCategory` (best for grouping) > `x_SkuMeterSubcategory` > `SkuMeter` +- **Commitment:** `CommitmentDiscountType` (Reservation/Savings Plan), `CommitmentDiscountStatus` (Used/Unused), `PricingCategory` (Standard/Committed/Dynamic). Utilization = Used/(Used+Unused). `x_CommitmentDiscountSavings` = savings per row. +- **Charge:** `ChargeCategory` (Usage/Purchase/Adjustment/Tax/Credit), `ChargeFrequency` (Usage-Based/Recurring/One-Time). Filter usage: `ChargeCategory == "Usage"`. + +## Tags + +Tags is dynamic JSON. Access: `Tags['tagname']`. Use `tostring()` when grouping: `by tostring(Tags['env'])`. + +## Query patterns + +See `query-catalog.md` for full templates. Key patterns: Top N (`top N by Cost desc`), Trend (`summarize by ChargePeriodStart`), Period comparison (`pivot(Period, sum(Cost))`). + +## Investigating changes ("why did cost go up/down?") + +- Compare FULL completed periods only. Never compare an incomplete current period against a full prior period. +- Filter anomaly analysis to `ChargeCategory == "Usage" and x_PublisherCategory == "Cloud Provider"`. +- Flag one-time purchases (`ChargeCategory == "Purchase"`) separately. +- If data partially covers the current period, say so explicitly. +- Diagnosis checklist for cost increases: + 1. Commitment coverage dropped? → RI/SP expired (check `CommitmentDiscountStatus == "Used"` share) + 2. `ListUnitPrice` changed? → Azure price increase + 3. `SkuMeter` or `ChargeDescription` changed? → resource resized + 4. None of the above → genuine usage increase + +## Response format + +1. **Quick answer** — 2-3 sentences with key finding and primary metric. +2. **Data** — formatted table. Never dump raw data inline. +3. **KQL query** — code block, separate from the answer. +4. **Recommendations** — actionable next steps if applicable. + +Always include the time range and scope of your analysis. If results seem incomplete or unexpected, say so. + +## KQL pitfalls + +- `nullif()` does NOT exist in KQL. Use `iff(x == 0, real(null), x)`. +- Division by zero: guard with `iff(denominator == 0, real(null), numerator / denominator)`. +- No empty lines within a query — they terminate execution. Use `//` comments for separation. +- `let` statements must end with `;`. Final expression must NOT. +- `max_of()`/`min_of()` for scalar comparison, not `max()`/`min()` (those are aggregation functions). +- Use `toreal()` before arithmetic on decimal columns to avoid type errors. +- Use explicit aliases: `sum(EffectiveCost)` → `Cost = sum(EffectiveCost)`. + +## Error handling + +- Schema error → check column name against schema reference, fix and retry. +- Timeout → add tighter time filters or reduce cardinality. +- Empty results → verify filters are not too restrictive, check data freshness. +- Retry up to 3 times before asking the user. diff --git a/src/templates/finops-hub-copilot-studio/knowledge-descriptions.md b/src/templates/finops-hub-copilot-studio/knowledge-descriptions.md new file mode 100644 index 000000000..d0a64265b --- /dev/null +++ b/src/templates/finops-hub-copilot-studio/knowledge-descriptions.md @@ -0,0 +1,15 @@ +# Knowledge file descriptions for Copilot Studio + +Use these descriptions when uploading knowledge files to your Copilot Studio agent. + +## schema-reference.md + +Column reference for building KQL queries against Costs_v1_2(). Contains all column names, data types, usage notes, and edge cases like blank meter categories and BilledCost vs EffectiveCost divergence. Use to look up correct column names before writing queries. + +## query-catalog.md + +Ready-to-use KQL query templates for FinOps analysis. Covers cost breakdowns by subscription/service/region, monthly trends, anomaly detection, forecasting, savings summary, commitment utilization, and reservation recommendations. Adapt these patterns to answer cost questions. + +## weekly-report-guide.md + +Step-by-step workflow for producing structured weekly cost anomaly reports. Contains 7 KQL queries (totals, category summary, resource increases/decreases, commitment coverage drops, marketplace), post-processing rules for grouping and severity classification, and the final report structure. diff --git a/src/templates/finops-hub-copilot-studio/knowledge/schema-reference.md b/src/templates/finops-hub-copilot-studio/knowledge/schema-reference.md new file mode 100644 index 000000000..9e7908f32 --- /dev/null +++ b/src/templates/finops-hub-copilot-studio/knowledge/schema-reference.md @@ -0,0 +1,298 @@ +# KQL column reference for Costs_v1_2() + +This document lists all columns available in the `Costs_v1_2()` function in the FinOps Hub database. + +**IMPORTANT: This is a SCHEMA REFERENCE for constructing KQL queries. Do NOT use values from this document to answer questions. Always EXECUTE a KQL query to get actual data.** + +Based on FOCUS 1.2 (FinOps Open Cost and Usage Specification). Columns with `x_` prefix are Azure extensions. + +## Time + +| Column | Type | Usage | +|--------|------|-------| +| ChargePeriodStart | datetime | Primary time filter. Daily granularity. | +| ChargePeriodEnd | datetime | Exclusive end of charge period. | +| BillingPeriodStart | datetime | Always 1st of month. Calendar-month filtering. | +| BillingPeriodEnd | datetime | Exclusive end. Use `<` not `<=`. | + +## Cost metrics + +| Column | Type | Usage | +|--------|------|-------| +| EffectiveCost | real | DEFAULT metric. Amortized cost after all discounts. | +| BilledCost | real | Actual invoice amount. Cash flow / accounts payable. | +| ListCost | real | Full retail, no discounts. | +| ContractedCost | real | Negotiated rate, before commitment discounts. | + +Relationship: `ListCost >= ContractedCost >= EffectiveCost` for discounted usage. + +### When BilledCost and EffectiveCost diverge + +| Scenario | BilledCost | EffectiveCost | +|----------|------------|---------------| +| On-demand usage (no discounts) | Same | Same | +| RI/SP covered usage | $0 | Daily amortized portion of prepayment | +| RI/SP unused portion | $0 | Daily amortized portion (waste) | +| RI/SP purchase (one-time) | Full purchase price | $0 (amortized to usage rows) | +| RI/SP purchase (monthly) | Monthly payment | $0 (amortized to usage rows) | + +## Organization hierarchy + +| Column | Type | Usage | +|--------|------|-------| +| BillingAccountId | string | Full ARM path of billing account. | +| BillingAccountName | string | Billing account display name. | +| BillingAccountType | string | e.g. "Billing Account". | +| BillingCurrency | string | Currency code (e.g. CAD, USD, EUR). | +| x_BillingAccountAgreement | string | Agreement type: "EA", "MCA", "MOSP". | +| x_BillingAccountId | string | Numeric billing account ID. | +| x_BillingProfileId | string | MCA billing profile ID. Not used for EA. | +| x_BillingProfileName | string | MCA billing profile name. | +| x_AccountId | string | EA enrollment account ID. | +| x_AccountName | string | EA enrollment account name. | +| x_AccountOwnerId | string | Email of EA enrollment account owner. | +| x_CustomerId | string | CSP customer tenant ID. | +| x_CustomerName | string | CSP customer name. | +| SubAccountId | string | Subscription ARM path. | +| SubAccountName | string | Subscription display name. | +| SubAccountType | string | e.g. "Subscription". | +| x_ResourceGroupName | string | Azure resource group name. | +| x_CostCenter | string | Custom chargeback value from EA/MCA. | +| x_CostAllocationRuleName | string | Cost allocation rule name. | +| x_CostCategories | dynamic | Cost categories from allocation rules. | +| x_Project | string | Project label from allocation or tags. | + +## Resource + +| Column | Type | Usage | +|--------|------|-------| +| ResourceId | string | Globally unique ARM path. Use for grouping/joins. | +| ResourceName | string | Display only — NOT unique across subscriptions. | +| ResourceType | string | Friendly type (e.g. "Virtual machine"). | +| x_ResourceType | string | ARM type (e.g. "Microsoft.Compute/virtualMachines"). | +| x_InstanceID | string | Legacy CM resource ID. Prefer ResourceId. | + +## Service hierarchy + +Service hierarchy: ServiceCategory > ServiceSubcategory > ServiceName > x_SkuMeterCategory > x_SkuMeterSubcategory > SkuMeter. FOCUS levels (first 3) don't align 1:1 with Azure meter levels (last 3). + +| Column | Type | Usage | +|--------|------|-------| +| ServiceCategory | string | FOCUS standard categories: AI and Machine Learning, Analytics, Business Applications, Compute, Containers, Databases, Developer Tools, Identity, Integration, Internet of Things, Management and Governance, Multicloud, Networking, Other, Security, Storage, Web. | +| ServiceSubcategory | string | FOCUS mid-level. Currently blank in Microsoft data. | +| ServiceName | string | FOCUS-mapped service name. | +| x_SkuMeterCategory | string | Best for service-level grouping. **Can be blank** — see below. | +| x_SkuMeterSubcategory | string | e.g. "Easv5 Series", "Standard SSD Managed Disks". | +| SkuMeter | string | Specific meter name. | +| x_SkuMeterId | string | Meter GUID. | +| ChargeDescription | string | Human-readable charge description. | +| x_ServiceCode | string | Internal Azure service code. | +| x_ServiceId | string | Internal Azure service ID. | +| x_ServiceModel | string | Deployment model: "IaaS", "PaaS", "SaaS". | + +### Blank x_SkuMeterCategory — expected, not a data issue + +| Charge type | Why blank | +|-------------|-----------| +| Savings Plan unused | SP is flexible, not tied to a specific service | +| Third-party vendor (`x_PublisherCategory == "Vendor"`) | Marketplace products lack Azure meter categorization | +| Azure Support | Account-level charge, no meter | + +To handle in queries, use a synthetic category: + +```kusto +| extend Category = iif(isempty(x_SkuMeterCategory), + iif(x_PublisherCategory == "Vendor", strcat("Vendor: ", PublisherName), + iif(CommitmentDiscountStatus == "Unused" and CommitmentDiscountType == "Savings Plan", "Savings Plan Unused", + iif(isnotempty(ServiceName), ServiceName, + "Uncategorized"))), + x_SkuMeterCategory) +``` + +## Commitment discounts + +| Column | Type | Usage | +|--------|------|-------| +| CommitmentDiscountType | string | "Reservation" or "Savings Plan". | +| CommitmentDiscountStatus | string | "Used" or "Unused". | +| CommitmentDiscountCategory | string | "Usage" (reservations) or "Spend" (savings plans). | +| CommitmentDiscountId | string | ARM path of reservation or savings plan. | +| CommitmentDiscountName | string | Display name. | +| CommitmentDiscountQuantity | real | Committed quantity. | +| CommitmentDiscountUnit | string | Unit for CommitmentDiscountQuantity. | +| x_CommitmentDiscountSavings | real | Savings vs on-demand price for this row. | +| x_CommitmentDiscountPercent | real | Discount percentage vs on-demand. | +| x_CommitmentDiscountNormalizedRatio | real | Normalization factor across VM sizes. | +| x_CommitmentDiscountSpendEligibility | string | Savings Plan eligibility. | +| x_CommitmentDiscountUsageEligibility | string | Reservation eligibility. | +| x_CommitmentDiscountUtilizationAmount | real | Utilized amount. | +| x_CommitmentDiscountUtilizationPotential | real | Maximum potential utilization. | +| x_AmortizationClass | string | "Amortized Charge" or "Principal". | + +## Capacity reservation + +| Column | Type | Usage | +|--------|------|-------| +| CapacityReservationId | string | ARM path of capacity reservation. | +| CapacityReservationStatus | string | "Used" or "Unused". | + +## Charge classification + +| Column | Type | Usage | +|--------|------|-------| +| ChargeCategory | string | Usage, Purchase, Adjustment, Tax, Credit. | +| ChargeClass | string | Empty = normal. "Correction" = adjusts prior period. | +| ChargeFrequency | string | Usage-Based, Recurring, One-Time. | + +## Pricing + +| Column | Type | Usage | +|--------|------|-------| +| PricingCategory | string | Standard (on-demand), Committed, Dynamic (spot), Other. | +| x_PricingSubcategory | string | Standard, Tiered, Committed Usage, Committed Spend, Spot. | +| ConsumedQuantity | real | Amount consumed in ConsumedUnit. | +| ConsumedUnit | string | Unit of measure for ConsumedQuantity. | +| PricingQuantity | real | Quantity used for pricing. | +| PricingUnit | string | FOCUS-normalized unit. | +| PricingCurrency | string | Currency used for pricing. | +| ListUnitPrice | real | Published retail price per PricingUnit. | +| ContractedUnitPrice | real | Negotiated price per PricingUnit. | +| x_EffectiveUnitPrice | real | Actual unit price after all discounts. | +| x_BilledUnitPrice | real | Unit price on invoice. | +| x_PricingBlockSize | real | Block size for pricing. | +| x_PricingUnitDescription | string | Original CM unit description with block size. | + +## Publisher + +| Column | Type | Usage | +|--------|------|-------| +| x_PublisherCategory | string | "Cloud Provider" (first-party) or "Vendor" (marketplace). | +| PublisherName | string | Publisher name (e.g. "Microsoft"). | +| x_PublisherId | string | Publisher as billed by. | +| ProviderName | string | Always "Microsoft" for Azure. | +| InvoiceIssuerName | string | Entity that issued the invoice. | + +## Geography + +| Column | Type | Usage | +|--------|------|-------| +| RegionId | string | e.g. "canadacentral". | +| RegionName | string | e.g. "Canada Central". | +| AvailabilityZone | string | Availability zone, if applicable. | +| x_SkuRegion | string | SKU region from CM. Different format than RegionId. | + +## Tags + +| Column | Type | Usage | +|--------|------|-------| +| Tags | dynamic | JSON key-value pairs. Access: `Tags['env']`. | + +## SKU details + +| Column | Type | Usage | +|--------|------|-------| +| SkuId | string | FOCUS-normalized SKU identifier. | +| SkuPriceDetails | dynamic | FOCUS-normalized pricing metadata (JSON). | +| SkuPriceId | string | FOCUS-normalized price sheet identifier. | +| x_SkuDetails | dynamic | Original CM SKU metadata (JSON). | +| x_SkuTerm | int | Purchase term in months (e.g. 12, 36). | +| x_SkuTier | string | Pricing tier level. | +| x_SkuCoreCount | int | Number of vCPUs for compute SKUs. | +| x_SkuDescription | string | Human-readable SKU description. | +| x_SkuInstanceType | string | VM size (e.g. "Standard_E4as_v5"). | +| x_SkuIsCreditEligible | bool | Whether eligible for Azure credits. | +| x_SkuLicenseQuantity | int | Number of licenses included. | +| x_SkuLicenseStatus | string | License status. | +| x_SkuLicenseType | string | e.g. "Windows_Server", "RHEL". | +| x_SkuLicenseUnit | string | e.g. "vCPU", "Core". | +| x_SkuOfferId | string | Azure offer ID (e.g. "MS-AZR-0017P"). | +| x_SkuOperatingSystem | string | OS (e.g. "Windows", "Linux"). | +| x_SkuOrderId | string | Reservation/SP order ID. | +| x_SkuOrderName | string | Reservation/SP order name. | +| x_SkuPartNumber | string | Azure part number. | +| x_SkuPlanName | string | Marketplace plan name. | +| x_SkuServiceFamily | string | Service family (e.g. "Compute", "Storage"). | + +## Invoice + +| Column | Type | Usage | +|--------|------|-------| +| InvoiceId | string | FOCUS invoice identifier. | +| x_InvoiceIssuerId | string | Invoice issuer ID. | +| x_InvoiceSectionId | string | MCA invoice section / EA department ID. | +| x_InvoiceSectionName | string | MCA invoice section / EA department name. | +| x_BillingExchangeRate | real | Exchange rate: PricingCurrency to BillingCurrency. | +| x_BillingExchangeRateDate | datetime | Date exchange rate was determined. | +| x_BillingItemCode | string | Internal billing line item code. | +| x_BillingItemName | string | Internal billing line item name. | + +## USD equivalents + +| Column | Type | Usage | +|--------|------|-------| +| x_BilledCostInUsd | real | BilledCost converted to USD. | +| x_EffectiveCostInUsd | real | EffectiveCost converted to USD. | +| x_ListCostInUsd | real | ListCost converted to USD. | +| x_ContractedCostInUsd | real | ContractedCost converted to USD. | + +## Discounts and savings + +| Column | Type | Usage | +|--------|------|-------| +| x_Credits | dynamic | Credit adjustments (JSON). | +| x_Discount | dynamic | Discount details (JSON). | +| x_NegotiatedDiscountPercent | real | EA/MCA negotiated discount percentage. | +| x_NegotiatedDiscountSavings | real | Savings from negotiated rates. | +| x_TotalDiscountPercent | real | Total discount % (negotiated + commitment). | +| x_TotalSavings | real | Total savings vs list price. | +| x_CurrencyConversionRate | real | Currency conversion rate. | + +## Service period + +| Column | Type | Usage | +|--------|------|-------| +| x_ServicePeriodStart | datetime | Start of service consumption period. | +| x_ServicePeriodEnd | datetime | End of service consumption period. | + +## Partner + +| Column | Type | Usage | +|--------|------|-------| +| x_ResellerId | string | CSP reseller tenant ID. | +| x_ResellerName | string | CSP reseller name. | +| x_PartnerCreditApplied | string | Whether partner earned credit was applied. | +| x_PartnerCreditRate | string | PEC discount rate. | +| x_OwnerAccountID | string | Account owner for CSP scenarios. | + +## Commodity + +| Column | Type | Usage | +|--------|------|-------| +| x_CommodityCode | string | Internal commodity code. | +| x_CommodityName | string | Internal commodity name. | +| x_ComponentName | string | Sub-resource component name. | +| x_ComponentType | string | Sub-resource component type. | +| x_SubproductName | string | Sub-product within the service. | + +## Usage + +| Column | Type | Usage | +|--------|------|-------| +| x_ConsumedCoreHours | real | Total vCPU-hours consumed (compute only). | +| x_CostType | string | Internal cost type classification. | +| x_Operation | string | Specific operation or API action. | +| x_UsageType | string | Internal usage type identifier. | + +## Pipeline / ingestion + +| Column | Type | Usage | +|--------|------|-------| +| x_ChargeId | string | Unique identifier for this charge row. | +| x_ExportTime | datetime | When data was exported from Cost Management. | +| x_IngestionTime | datetime | When data was ingested into FinOps Hub. | +| x_SourceName | string | Data source/export name. | +| x_SourceProvider | string | Data provider. | +| x_SourceType | string | Type of data source. | +| x_SourceVersion | string | Schema version of source data. | +| x_SourceChanges | string | Changes applied during ingestion. | +| x_SourceValues | dynamic | Original source values before transformation (JSON). | diff --git a/src/templates/finops-hub-copilot-studio/knowledge/weekly-report-guide.md b/src/templates/finops-hub-copilot-studio/knowledge/weekly-report-guide.md new file mode 100644 index 000000000..0a16b36cd --- /dev/null +++ b/src/templates/finops-hub-copilot-studio/knowledge/weekly-report-guide.md @@ -0,0 +1,188 @@ +# Weekly cost anomaly report guide + +This guide defines the workflow for producing a structured weekly cost anomaly report. Follow these steps when the user asks for a weekly report, weekly summary, or anomaly review. + +**IMPORTANT: This is a workflow guide. Execute all queries via the Kusto MCP tool — never return this document as an answer.** + +## Date boundaries + +Use these for all queries in the report: + +```kusto +let LastWeekEnd = startofweek(now()); +let LastWeekStart = datetime_add('day', -7, LastWeekEnd); +let PriorWeekStart = datetime_add('day', -14, LastWeekEnd); +``` + +## Queries to run + +All queries filter `ChargeCategory == "Usage" and x_PublisherCategory == "Cloud Provider"` unless noted otherwise. + +### Q1: Weekly cost total + +```kusto +let LastWeekEnd = startofweek(now()); +let LastWeekStart = datetime_add('day', -7, LastWeekEnd); +let PriorWeekStart = datetime_add('day', -14, LastWeekEnd); +Costs_v1_2() +| where ChargePeriodStart >= PriorWeekStart and ChargePeriodStart < LastWeekEnd + and ChargeCategory == "Usage" and x_PublisherCategory == "Cloud Provider" +| summarize Cost = round(sum(EffectiveCost), 2), Days = dcount(ChargePeriodStart) + by Week = iff(ChargePeriodStart >= LastWeekStart, "LastWeek", "PriorWeek") +| extend DailyAvg = round(Cost / Days, 2) +``` + +### Q2: Category summary (top movers) + +```kusto +let LastWeekEnd = startofweek(now()); +let LastWeekStart = datetime_add('day', -7, LastWeekEnd); +let PriorWeekStart = datetime_add('day', -14, LastWeekEnd); +Costs_v1_2() +| where ChargePeriodStart >= PriorWeekStart and ChargePeriodStart < LastWeekEnd + and ChargeCategory == "Usage" and x_PublisherCategory == "Cloud Provider" +| extend Week = iff(ChargePeriodStart >= LastWeekStart, "LastWeek", "PriorWeek") +| summarize Cost = round(sum(EffectiveCost), 2) by x_SkuMeterCategory, Week +| evaluate pivot(Week, sum(Cost)) +| extend Change = round(LastWeek - PriorWeek, 2), + ChangePct = round((LastWeek - PriorWeek) / iff(PriorWeek == 0, real(null), PriorWeek) * 100, 1) +| where abs(Change) > 50 or abs(ChangePct) > 20 +| order by Change desc +``` + +### Q3: Resource increases + +```kusto +let LastWeekEnd = startofweek(now()); +let LastWeekStart = datetime_add('day', -7, LastWeekEnd); +let PriorWeekStart = datetime_add('day', -14, LastWeekEnd); +Costs_v1_2() +| where ChargePeriodStart >= PriorWeekStart and ChargePeriodStart < LastWeekEnd + and ChargeCategory == "Usage" and x_PublisherCategory == "Cloud Provider" +| extend Week = iff(ChargePeriodStart >= LastWeekStart, "LastWeek", "PriorWeek") +| summarize Cost = round(sum(EffectiveCost), 2) + by ResourceId, ResourceName, x_ResourceGroupName, x_SkuMeterCategory, Week +| evaluate pivot(Week, sum(Cost)) +| extend Change = round(LastWeek - PriorWeek, 2), + ChangePct = round((LastWeek - PriorWeek) / iff(PriorWeek == 0, real(null), PriorWeek) * 100, 1) +| where LastWeek > 50 and (ChangePct > 50 or Change > 100) +| order by Change desc +``` + +### Q4: Resource decreases + +Same as Q3 but filter for decreases: + +```kusto +let LastWeekEnd = startofweek(now()); +let LastWeekStart = datetime_add('day', -7, LastWeekEnd); +let PriorWeekStart = datetime_add('day', -14, LastWeekEnd); +Costs_v1_2() +| where ChargePeriodStart >= PriorWeekStart and ChargePeriodStart < LastWeekEnd + and ChargeCategory == "Usage" and x_PublisherCategory == "Cloud Provider" +| extend Week = iff(ChargePeriodStart >= LastWeekStart, "LastWeek", "PriorWeek") +| summarize Cost = round(sum(EffectiveCost), 2) + by ResourceId, ResourceName, x_ResourceGroupName, x_SkuMeterCategory, Week +| evaluate pivot(Week, sum(Cost)) +| extend Change = round(LastWeek - PriorWeek, 2), + ChangePct = round((LastWeek - PriorWeek) / iff(PriorWeek == 0, real(null), PriorWeek) * 100, 1) +| where PriorWeek > 50 and (ChangePct < -50 or Change < -100) +| order by Change asc +``` + +### Q5: Commitment coverage drops + +```kusto +let LastWeekEnd = startofweek(now()); +let LastWeekStart = datetime_add('day', -7, LastWeekEnd); +let PriorWeekStart = datetime_add('day', -14, LastWeekEnd); +Costs_v1_2() +| where ChargePeriodStart >= PriorWeekStart and ChargePeriodStart < LastWeekEnd +| extend Week = iff(ChargePeriodStart >= LastWeekStart, "LastWeek", "PriorWeek") +| summarize + Total = round(sum(EffectiveCost), 2), + RICost = round(sumif(EffectiveCost, CommitmentDiscountStatus == "Used"), 2), + Savings = round(sum(x_CommitmentDiscountSavings), 2) + by ResourceId, ResourceName, x_ResourceGroupName, x_SkuMeterCategory, Week +| extend CovPct = round(RICost / iff(Total == 0, real(null), Total) * 100, 1) +| evaluate pivot(Week, take_any(Total), take_any(RICost), take_any(CovPct), take_any(Savings)) +| where PriorWeek_CovPct > 30 and PriorWeek_Total > 50 + and (PriorWeek_CovPct - LastWeek_CovPct) > 30 +| extend MonthlySavingsAtRisk = round((PriorWeek_Savings - LastWeek_Savings) * 4.33, 2) +| order by MonthlySavingsAtRisk desc +``` + +### Q6: Marketplace costs + +```kusto +let LastWeekEnd = startofweek(now()); +let LastWeekStart = datetime_add('day', -7, LastWeekEnd); +let PriorWeekStart = datetime_add('day', -14, LastWeekEnd); +Costs_v1_2() +| where ChargePeriodStart >= PriorWeekStart and ChargePeriodStart < LastWeekEnd + and x_PublisherCategory == "Vendor" +| extend Week = iff(ChargePeriodStart >= LastWeekStart, "LastWeek", "PriorWeek") +| summarize Cost = round(sum(EffectiveCost), 2) + by ResourceId, ResourceName, x_ResourceGroupName, PublisherName, ChargeCategory, Week +| evaluate pivot(Week, sum(Cost)) +| extend LastWeek = coalesce(LastWeek, 0.0), PriorWeek = coalesce(PriorWeek, 0.0) +| extend Change = round(LastWeek - PriorWeek, 2), + ChangePct = round((LastWeek - PriorWeek) / iff(PriorWeek == 0, real(null), PriorWeek) * 100, 1) +| where LastWeek > 10 or PriorWeek > 10 +| order by Change desc +``` + +### Q7: Marketplace purchases (13-month lookback) + +```kusto +Costs_v1_2() +| where ChargePeriodStart >= datetime_add('month', -13, startofmonth(now())) + and ChargeCategory == "Purchase" and x_PublisherCategory == "Vendor" +| summarize Cost = round(sum(EffectiveCost), 2) + by BillingMonth = startofmonth(ChargePeriodStart), PublisherName +| where Cost >= 10 +| order by PublisherName asc, BillingMonth asc +``` + +Classify each publisher by frequency: 10+ months = Monthly recurring, 3-9 months = Intermittent, 2 months = Annual, 1 month (current) = New, 1 month (older) = One-time. + +## Post-processing rules + +### Cross-reference coverage drops with increases + +If a ResourceId appears in both Q3 (increases) and Q5 (coverage drops): +- Remove it from the increases table. +- Label it "RI/SP expired" in the coverage section. +- Add a summary note: "N resources saw +$X/week due to expired commitments." + +### Group by resource group + +- 5+ resources with same direction in same RG → consolidate to one line: "~N resources in `rg-name`, total +$X/week" +- 3-4 resources with same pattern → shared subheading +- Fewer than 3 or mixed direction → list individually + +### Severity classification + +**New resources** (PriorWeek is null or 0): +- Major: >$200/week +- Minor: >$50/week +- Watch: below $50/week + +**Existing resources:** +- Major: abs(Change) > $200, OR abs(ChangePct) > 100% AND LastWeek > $100 +- Minor: abs(Change) > $50 +- Watch: below thresholds + +Show Major and Minor in main tables. Collect Watch items in a separate sub-table. + +## Report structure + +Present the report in this order: + +1. **Total cost summary** — last week vs prior week, daily average, overall change +2. **Category summary** — top movers by service category with key driver bullets +3. **Commitment coverage changes** — if any; include prior/current coverage %, monthly savings at risk, recommendation to renew +4. **Major increases** — resource-level, grouped by RG where applicable +5. **Significant decreases** — resource-level, grouped by RG +6. **Marketplace** — usage changes + purchases classified by pattern (monthly/intermittent/new/one-time) +7. **Action items** — prioritized list of recommended follow-ups diff --git a/src/templates/finops-hub/.build.config b/src/templates/finops-hub/.build.config index 11a298117..2dd495e60 100644 --- a/src/templates/finops-hub/.build.config +++ b/src/templates/finops-hub/.build.config @@ -1,4 +1,7 @@ { + "scripts": [ + "Build-HubIngestionQueries.ps1" + ], "ignore": [ "errors.json", "modules/README.md", diff --git a/src/templates/finops-hub/createUiDefinition.json b/src/templates/finops-hub/createUiDefinition.json index d16cbab0a..da1f029b8 100644 --- a/src/templates/finops-hub/createUiDefinition.json +++ b/src/templates/finops-hub/createUiDefinition.json @@ -166,11 +166,11 @@ "name": "remoteHubStorageUri", "type": "Microsoft.Common.TextBox", "label": "Remote hub storage URI", - "toolTip": "Data Lake storage endpoint from the remote hub storage account. Copy from the storage account Settings > Endpoints > Data Lake storage. Example: https://myremotehub.dfs.core.windows.net/", + "toolTip": "Data Lake storage endpoint from the remote hub storage account. Copy from the storage account Settings > Endpoints > Data Lake storage. Example: https://myremotehub.dfs.core.windows.net/ (endpoint suffix varies by cloud)", "constraints": { "required": "[equals(basics('analyticsBackend').analyticsEngine, 'remote')]", - "regex": "^https://.*\\.dfs\\.core\\.windows\\.net/?$", - "validationMessage": "Must be a valid Data Lake storage endpoint URL in the format: https://storageaccount.dfs.core.windows.net/" + "regex": "^https://.*\\.dfs\\.core\\.[a-z.]+/?$", + "validationMessage": "Must be a valid Data Lake storage endpoint URL (e.g., https://storageaccount.dfs.core.windows.net/ for Azure public cloud)" }, "visible": "[equals(basics('analyticsBackend').analyticsEngine, 'remote')]" }, @@ -701,6 +701,141 @@ } ] }, + { + "name": "recommendations", + "label": "🆕 Recommendations", + "elements": [ + { + "name": "recommendationsIntro", + "type": "Microsoft.Common.TextBlock", + "visible": true, + "options": { + "text": "Uncover hidden savings with FinOps hubs recommendations. FinOps hubs can automatically scan your environment using Azure Resource Graph to surface cost optimization opportunities like idle resources and missed discounts that aren't available in Microsoft Cost Management or Azure Advisor." + } + }, + { + "name": "enableRecommendations", + "type": "Microsoft.Common.CheckBox", + "label": "Enable hubs recommendations (preview)" + }, + { + "name": "included", + "type": "Microsoft.Common.Section", + "label": "Included recommendations", + "visible": "[steps('recommendations').enableRecommendations]", + "elements": [ + { + "name": "includedIntro", + "type": "Microsoft.Common.TextBlock", + "visible": true, + "options": { + "text": "The following recommendations are always included when enabled:" + } + }, + { + "name": "advisorCost", + "type": "Microsoft.Common.TextBlock", + "visible": true, + "options": { + "text": "✅ Azure Advisor cost recommendations" + } + }, + { + "name": "stoppedVMs", + "type": "Microsoft.Common.TextBlock", + "visible": true, + "options": { + "text": "✅ Stopped (not deallocated) VMs" + } + }, + { + "name": "unattachedDisks", + "type": "Microsoft.Common.TextBlock", + "visible": true, + "options": { + "text": "✅ Unattached managed disks" + } + }, + { + "name": "unattachedPublicIPs", + "type": "Microsoft.Common.TextBlock", + "visible": true, + "options": { + "text": "✅ Unattached static public IPs" + } + }, + { + "name": "emptySQLElasticPools", + "type": "Microsoft.Common.TextBlock", + "visible": true, + "options": { + "text": "✅ Empty SQL elastic pools" + } + }, + { + "name": "backendlessAppGateways", + "type": "Microsoft.Common.TextBlock", + "visible": true, + "options": { + "text": "✅ Application Gateways with empty backend pools" + } + }, + { + "name": "backendlessLoadBalancers", + "type": "Microsoft.Common.TextBlock", + "visible": true, + "options": { + "text": "✅ Load Balancers without backends" + } + } + ] + }, + { + "name": "optional", + "type": "Microsoft.Common.Section", + "label": "Optional recommendations", + "visible": "[steps('recommendations').enableRecommendations]", + "elements": [ + { + "name": "optionalIntro", + "type": "Microsoft.Common.TextBlock", + "visible": true, + "options": { + "text": "The following recommendations do not apply in all cases and are optional:" + } + }, + { + "name": "enableAHBRecommendations", + "type": "Microsoft.Common.CheckBox", + "label": "➕ Azure Hybrid Benefit", + "toolTip": "Flag VMs and SQL VMs without Azure Hybrid Benefit enabled. May generate noise if your organization does not have on-premises licenses." + }, + { + "name": "enableSpotRecommendations", + "type": "Microsoft.Common.CheckBox", + "label": "➕ Non-Spot AKS cluster", + "toolTip": "Flag AKS clusters with autoscaling but not using Spot VMs. May generate noise since Spot VMs are only appropriate for interruptible workloads." + } + ] + }, + { + "name": "permissions", + "type": "Microsoft.Common.Section", + "label": "Required permissions", + "visible": "[steps('recommendations').enableRecommendations]", + "elements": [ + { + "name": "permissionsNote", + "type": "Microsoft.Common.TextBlock", + "visible": true, + "options": { + "text": "The Data Factory managed identity requires Reader role on the management groups or subscriptions you want to scan. After deployment, grant the hub's managed identity Reader access to the desired scopes." + } + } + ] + } + ] + }, { "name": "advanced", "label": "Advanced", @@ -838,6 +973,9 @@ "fabricCapacityUnits": "[if(equals(basics('analyticsBackend').analyticsEngine, 'fabric'), basics('analyticsBackend').fabricCapacity, '')]", "enableInfrastructureEncryption": "[steps('advanced').storage.enableInfrastructureEncryption]", "enableManagedExports": "[steps('advanced').managedExports.enableManagedExports]", + "enableRecommendations": "[steps('recommendations').enableRecommendations]", + "enableAHBRecommendations": "[steps('recommendations').optional.enableAHBRecommendations]", + "enableSpotRecommendations": "[steps('recommendations').optional.enableSpotRecommendations]", "enablePublicAccess": "[steps('advanced').networking.enablePublicAccess]", "virtualNetworkAddressPrefix": "[steps('advanced').networking.virtualNetworkAddressPrefix]", "dataExplorerSku": "[steps('pricing').dataExplorer.dataExplorerSku]", diff --git a/src/templates/finops-hub/main.bicep b/src/templates/finops-hub/main.bicep index 033b19e6f..4e058fcba 100644 --- a/src/templates/finops-hub/main.bicep +++ b/src/templates/finops-hub/main.bicep @@ -39,6 +39,15 @@ param remoteHubStorageKey string = '' @description('Optional. Enable managed exports where your FinOps hub instance will create and run Cost Management exports on your behalf. Not supported for Microsoft Customer Agreement (MCA) billing profiles. Requires the ability to grant User Access Administrator role to FinOps hubs, which is required to create Cost Management exports. Default: true.') param enableManagedExports bool = true +@description('Optional. Enable recommendations ingested from Azure Resource Graph based on configurable queries. The Data Factory managed identity requires Reader role on management groups or subscriptions to execute Resource Graph queries. Default: false.') +param enableRecommendations bool = false + +@description('Optional. Enable Azure Hybrid Benefit recommendations that flag VMs and SQL VMs without Azure Hybrid Benefit enabled. May generate noise if your organization does not have on-premises licenses. Requires enableRecommendations. Default: false.') +param enableAHBRecommendations bool = false + +@description('Optional. Enable non-Spot AKS cluster recommendations that flag AKS clusters with autoscaling but not using Spot VMs. May generate noise since Spot VMs are only appropriate for interruptible workloads. Requires enableRecommendations. Default: false.') +param enableSpotRecommendations bool = false + @description('Optional. Name of the Azure Data Explorer cluster to use for advanced analytics. If empty, Azure Data Explorer will not be deployed. Required to use with Power BI if you have more than $2-5M/mo in costs being monitored. Default: "" (do not use).') param dataExplorerName string = '' @@ -168,6 +177,9 @@ module hub 'modules/hub.bicep' = { enableInfrastructureEncryption: enableInfrastructureEncryption enablePurgeProtection: enablePurgeProtection enableManagedExports: enableManagedExports + enableRecommendations: enableRecommendations + enableAHBRecommendations: enableAHBRecommendations + enableSpotRecommendations: enableSpotRecommendations dataExplorerName: dataExplorerName dataExplorerSku: dataExplorerSku dataExplorerCapacity: dataExplorerCapacity diff --git a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/app.bicep b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/app.bicep index b880e7108..46b5ea819 100644 --- a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/app.bicep +++ b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/app.bicep @@ -143,7 +143,13 @@ var useFabric = !empty(fabricQueryUri) var useAzure = !useFabric && !empty(clusterName) // cSpell:ignore ftkver, privatelink -var dataExplorerPrivateDnsZoneName = replace('privatelink.${app.hub.location}.${replace(environment().suffixes.storage, 'core', 'kusto')}', '..', '.') +var dataExplorerDnsSuffixLookup = { + AzureCloud: 'kusto.windows.net' + AzureUSGovernment: 'kusto.usgovcloudapi.net' + AzureChinaCloud: 'kusto.windows.cn' +} +var dataExplorerDnsSuffix = dataExplorerDnsSuffixLookup[?environment().name] ?? replace(environment().suffixes.storage, 'core', 'kusto') +var dataExplorerPrivateDnsZoneName = replace('privatelink.${app.hub.location}.${dataExplorerDnsSuffix}', '..', '.') // Actual = Minimum(ClusterMaximumConcurrentOperations, Number of nodes in cluster * Maximum(1, Core count per node * CoreUtilizationCoefficient)) var ingestionCapacity = { @@ -219,7 +225,7 @@ var dataExplorerIngestionCapacity = useFabric // WORKAROUND: Direct property access fails on cluster updates due to ARM bug // See: https://github.com/Azure/azure-resource-manager-templates/issues/[issue-number] -var dataExplorerUri = useFabric ? fabricQueryUri : 'https://${cluster.name}.${app.hub.location}.kusto.windows.net' +var dataExplorerUri = useFabric ? fabricQueryUri : 'https://${cluster.name}.${app.hub.location}.${dataExplorerDnsSuffix}' //============================================================================== // Resources @@ -546,7 +552,7 @@ resource dataFactoryVNet 'Microsoft.DataFactory/factories/managedVirtualNetworks #disable-next-line BCP318 // Null safety warning for conditional resource access // Null safety warning for conditional resource access // Null safety warning for conditional resource access privateLinkResourceId: cluster.id fqdns: [ - 'https://${replace(clusterName, '_', '-')}.${app.hub.location}.kusto.windows.net' + 'https://${replace(clusterName, '_', '-')}.${app.hub.location}.${dataExplorerDnsSuffix}' ] } } @@ -594,19 +600,14 @@ resource linkedService_dataExplorer 'Microsoft.DataFactory/factories/linkedservi } } -// GitHub repository linked service for FTK open data +// GitHub repository linked service for FTK release files resource linkedService_ftkRepo 'Microsoft.DataFactory/factories/linkedservices@2018-06-01' = { name: 'ftkRepo' parent: dataFactory properties: { type: 'HttpServer' - parameters: { - filePath: { - type: 'string' - } - } typeProperties: { - url: '@concat(\'https://gitapp.hub.com/microsoft/finops-toolkit/\', linkedService().filePath)' + url: 'https://github.com/microsoft/finops-toolkit/' enableServerCertificateValidation: true authenticationType: 'Anonymous' } diff --git a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/HubSetup_v1_2.kql b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/HubSetup_v1_2.kql index 65775bb04..ffc8d5de5 100644 --- a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/HubSetup_v1_2.kql +++ b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/HubSetup_v1_2.kql @@ -462,13 +462,21 @@ Recommendations_v1_2() ) | project ProviderName, + ResourceId, + ResourceName, + ResourceType, SubAccountId, - x_IngestionTime, + SubAccountName, x_EffectiveCostAfter, x_EffectiveCostBefore, x_EffectiveCostSavings, + x_IngestionTime, + x_RecommendationCategory, x_RecommendationDate, + x_RecommendationDescription, x_RecommendationDetails, + x_RecommendationId, + x_ResourceGroupName, x_SourceName, x_SourceProvider, x_SourceType, diff --git a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_HubInfra.kql b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_HubInfra.kql index dfb53d07e..19c907ec8 100644 --- a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_HubInfra.kql +++ b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_HubInfra.kql @@ -98,7 +98,8 @@ parse_resourceid(resourceId: string) { // let ResourceId = tolower('/providers/Microsoft.BillingBenefits/savingsPlanOrders/2d2e284b-0638-427e-b8c6-1b874d4f17c8/sp/xxx'); let SubAccountId = tostring(extract('/subscriptions/[^/]+', 1, ResourceId)); let x_ResourceGroupName = tostring(extract('/resourcegroups/[^/]+', 1, ResourceId)); - let providerPath = iff(ResourceId !contains '/providers/', '', split(iff(ResourceId startswith '/subscriptions/', strcat('/providers/microsoft.resources/', ResourceId), ResourceId), '/providers/')[-1]); + let tmp_ResourceId = iff(ResourceId !contains '/providers/' and ResourceId startswith '/subscriptions/', strcat('/providers/microsoft.resources', ResourceId), ResourceId); + let providerPath = iff(tmp_ResourceId !contains '/providers/', '', tostring(split(tmp_ResourceId, '/providers/')[-1])); let x_ResourceProvider = iff(isempty(providerPath), '', split(providerPath, '/')[0]); let tmp_ResourceProviderPath = iff(isempty(providerPath), '', substring(providerPath, strlen(x_ResourceProvider) + 1)); let segments = split(tmp_ResourceProviderPath, '/'); diff --git a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_v1_2.kql b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_v1_2.kql index 2b36351ea..daeab5527 100644 --- a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_v1_2.kql +++ b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_v1_2.kql @@ -1685,6 +1685,18 @@ Recommendations_transform_v1_2() | extend NetSavings = case(isnotempty(NetSavings), NetSavings, isnotempty(NetSavingsJson), toreal(extract(@'"value":([0-9\.]+)', 1, NetSavingsJson)), NetSavings) | extend TotalCostWithReservedInstances = case(isnotempty(TotalCostWithReservedInstances), TotalCostWithReservedInstances, isnotempty(TotalCostWithReservedInstancesJson), toreal(extract(@'"value":([0-9\.]+)', 1, TotalCostWithReservedInstancesJson)), TotalCostWithReservedInstances) // + // Parse x_RecommendationDetails from JSON string (ARG queries serialize to string for Parquet compatibility) + | extend x_RecommendationDetails = iff(gettype(x_RecommendationDetails) == 'string', parse_json(tostring(x_RecommendationDetails)), x_RecommendationDetails) + // + // Normalize x_RecommendationDetails keys to x_PascalCase (Advisor extendedProperties use camelCase) + // Guard: inject a placeholder key so mv-apply doesn't drop rows with null/empty bags + | extend x_RecommendationDetails = bag_merge(coalesce(x_RecommendationDetails, dynamic({})), bag_pack('__placeholder', '')) + | mv-apply k = bag_keys(x_RecommendationDetails) on ( + where isnotempty(tostring(k)) and tostring(k) != '__placeholder' + | extend newKey = iff(tostring(k) startswith 'x_', tostring(k), strcat('x_', toupper(substring(tostring(k), 0, 1)), substring(tostring(k), 1))) + | summarize x_RecommendationDetails = make_bag(bag_pack(newKey, x_RecommendationDetails[tostring(k)])) + ) + // // Build recommendation details | lookup kind=leftouter (Regions | summarize RegionName = make_set(RegionName)[0] by Location = RegionId) on Location | extend x_RecommendationDetails = case( @@ -1719,23 +1731,30 @@ Recommendations_transform_v1_2() | extend x_RecommendationDate = coalesce(x_RecommendationDate, FirstUsageDate + (toint(extract(@'^P([0-9]+)D$', 1, tostring(x_RecommendationDetails.LookbackPeriodDuration))) * 1d)) | extend x_RecommendationDate = iff(x_RecommendationDate > now(), startofday(now()), x_RecommendationDate) // + // Derive x_ResourceType from ResourceId + | extend tmp_ResourceType = tostring(parse_resourceid(ResourceId).x_ResourceType) + | extend x_RecommendationDetails = iff(isnotempty(tmp_ResourceType), bag_merge(bag_pack('x_ResourceType', tmp_ResourceType), x_RecommendationDetails), x_RecommendationDetails) + // + // Set ResourceType display name from x_ResourceType code + | extend ResourceType = coalesce(ResourceType, tostring(resource_type(tmp_ResourceType).SingularDisplayName), tmp_ResourceType) + // | project ProviderName, - ResourceId, - ResourceName, + ResourceId = tolower(ResourceId), // Force lowercase for consistent grouping/filtering + ResourceName = tolower(iff(tmp_ResourceType =~ 'microsoft.resources/subscriptions', coalesce(SubAccountName, ResourceName), ResourceName)), // Use subscription name for subscription-level recommendations ResourceType, SubAccountId = coalesce(SubAccountId, iff(isnotempty(SubscriptionId), strcat('/subscriptions/', SubscriptionId), '')), SubAccountName, x_EffectiveCostAfter = coalesce(x_EffectiveCostAfter, TotalCostWithReservedInstances), x_EffectiveCostBefore = coalesce(x_EffectiveCostBefore, CostWithNoReservedInstances), - x_EffectiveCostSavings = coalesce(x_EffectiveCostSavings, NetSavings), + x_EffectiveCostSavings = coalesce(x_EffectiveCostSavings, NetSavings, toreal(x_RecommendationDetails.x_SavingsAmount), toreal(x_RecommendationDetails.x_AnnualSavingsAmount) / 12), x_IngestionTime, x_RecommendationCategory, // TODO: Set for reservation recommendations x_RecommendationDate, - x_RecommendationDescription, + x_RecommendationDescription = coalesce(x_RecommendationDescription, tostring(x_RecommendationDetails.x_RecommendationSolution), tostring(x_RecommendationDetails.x_RecommendationSubCategory)), x_RecommendationDetails, - x_RecommendationId, // TODO: Set for reservation recommendations - x_ResourceGroupName, + x_RecommendationId = tolower(x_RecommendationId), // TODO: Set for reservation recommendations; force lowercase for consistent grouping/filtering + x_ResourceGroupName = tolower(x_ResourceGroupName), // Force lowercase for consistent grouping/filtering x_SourceName, x_SourceProvider, x_SourceType, diff --git a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/AzureResourceGraph/README.md b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/AzureResourceGraph/README.md new file mode 100644 index 000000000..ab0b817d3 --- /dev/null +++ b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/AzureResourceGraph/README.md @@ -0,0 +1,24 @@ +# AzureResourceGraph engine app + +Query engine for Azure Resource Graph (ARG). Implements the `queries_{engineName}_ExecuteQuery` contract for the IngestionQueries orchestrator. + +## What it provides + +- **`azureResourceGraph` dataset** — ADF REST dataset pointing to the ARG API (`/providers/Microsoft.ResourceGraph/resources?api-version=2022-10-01`) +- **`queries_ResourceGraph_ExecuteQuery` pipeline** — Executes a single ARG query via REST POST and writes results as Parquet to the ingestion container + +## How it works + +1. IngestionQueries dispatches to this pipeline via the ADF REST API +2. The pipeline POSTs the query to the ARG endpoint, appending source metadata columns (`x_SourceName`, `x_SourceType`, `x_SourceProvider`, `x_SourceVersion`) directly in the query text +3. ARG returns results as JSON; the ADF Copy activity uses the provided `translator` to map columns and writes Parquet to the `ingestionPath` + +## Dependencies + +- **Core app** — provides the `azurerm` linked service (REST service with MSI auth to ARM) and the `ingestion` dataset +- **Data Factory managed identity** — must have **Reader** role on the tenant root management group (or individual subscriptions/management groups) to execute ARG queries across the tenant + +## Limitations + +- ARG queries are limited to 1,000 rows per page. The current implementation does not paginate, so queries returning more than 1,000 rows will be truncated. Pagination support can be added later if needed. +- ARG query text has a 10 KB limit. diff --git a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/AzureResourceGraph/app.bicep b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/AzureResourceGraph/app.bicep new file mode 100644 index 000000000..5ebd98b1f --- /dev/null +++ b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/AzureResourceGraph/app.bicep @@ -0,0 +1,262 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { finOpsToolkitVersion, HubAppProperties, isSupportedVersion } from '../../fx/hub-types.bicep' +import { AppMetadata as CoreMetadata } from '../Core/metadata.bicep' +import { AppMetadata as AzureResourceGraphMetadata } from './metadata.bicep' + +metadata hubApp = { + id: 'Microsoft.FinOpsHubs.AzureResourceGraph' + version: '$$ftkver$$' + dependencies: [ + 'Microsoft.FinOpsHubs.Core' + 'Microsoft.FinOpsHubs.IngestionQueries' + ] + metadata: 'https://microsoft.github.io/finops-toolkit/deploy/$$ftkver$$/Microsoft.FinOpsHubs/AzureResourceGraph/metadata.bicep' +} + + +//============================================================================== +// Parameters +//============================================================================== + +@description('Required. FinOps hub app getting deployed.') +param app HubAppProperties + +@description('Required. Metadata describing shared resources from the Core app. Must be v13 or higher.') +@validate(x => isSupportedVersion(x.version, '13.0', ''), 'Core app version must be 13.0 or higher.') +param core CoreMetadata + + + +//============================================================================== +// Variables +//============================================================================== + + + +//============================================================================== +// Resources +//============================================================================== + +// Register app +module appRegistration '../../fx/hub-app.bicep' = { + name: 'Microsoft.FinOpsHubs.AzureResourceGraph_Register' + params: { + app: app + version: finOpsToolkitVersion + features: [ + 'DataFactory' // ARG dataset and engine pipeline + ] + } +} + +// Get data factory instance +resource dataFactory 'Microsoft.DataFactory/factories@2018-06-01' existing = { + name: app.dataFactory + dependsOn: [appRegistration] +} + +//------------------------------------------------------------------------------ +// Datasets +//------------------------------------------------------------------------------ + +// Reference the ARM linked service (created by the Core app) +resource linkedService_arm 'Microsoft.DataFactory/factories/linkedservices@2018-06-01' existing = { + name: core.linkedServices.azurerm + parent: dataFactory +} + +// Resource Graph dataset - points to the ARG REST API +resource dataset_azureResourceGraph 'Microsoft.DataFactory/factories/datasets@2018-06-01' = { + name: 'azureResourceGraph' + parent: dataFactory + properties: { + annotations: [] + parameters: {} + type: 'RestResource' + typeProperties: { + relativeUrl: '/providers/Microsoft.ResourceGraph/resources?api-version=2022-10-01' + } + linkedServiceName: { + parameters: {} + referenceName: linkedService_arm.name + type: 'LinkedServiceReference' + } + } +} + +// Reference existing ingestion dataset from Core app +resource dataset_ingestion 'Microsoft.DataFactory/factories/datasets@2018-06-01' existing = { + name: core.datasets.ingestion + parent: dataFactory +} + +//------------------------------------------------------------------------------ +// Engine pipeline +//------------------------------------------------------------------------------ + +resource pipeline_ExecuteQuery 'Microsoft.DataFactory/factories/pipelines@2018-06-01' = { + name: 'queries_ResourceGraph_ExecuteQuery' + parent: dataFactory + properties: { + activities: [ + { + name: 'Check Query Has Results' + description: 'Run the query with | count to check if there are any results before attempting the full copy.' + type: 'WebActivity' + dependsOn: [] + policy: { + timeout: '0.00:05:00' + retry: 1 + retryIntervalInSeconds: 30 + secureOutput: false + secureInput: false + } + userProperties: [] + typeProperties: { + url: '${environment().resourceManager}providers/Microsoft.ResourceGraph/resources?api-version=2022-10-01' + method: 'POST' + headers: { + 'Content-Type': 'application/json' + } + body: { + value: '@concat(\'{ "query": "\', pipeline().parameters.query, \' | count" }\')' + type: 'Expression' + } + authentication: { + type: 'MSI' + resource: environment().resourceManager + } + } + } + { + name: 'If Query Has Results' + description: 'Only run the copy if the query returned results to avoid schema mapping errors on empty result sets.' + type: 'IfCondition' + dependsOn: [ + { + activity: 'Check Query Has Results' + dependencyConditions: ['Succeeded'] + } + ] + userProperties: [] + typeProperties: { + expression: { + value: '@greater(int(activity(\'Check Query Has Results\').output.data[0].Count), 0)' + type: 'Expression' + } + ifTrueActivities: [ + { + name: 'Execute ARG Query' + description: 'Execute a single ARG query and write the result to the ingestion container as Parquet.' + type: 'Copy' + dependsOn: [] + policy: { + timeout: '0.00:10:00' + retry: 0 + retryIntervalInSeconds: 60 + secureOutput: false + secureInput: false + } + userProperties: [] + typeProperties: { + source: { + type: 'RestSource' + httpRequestTimeout: '00:02:00' + requestInterval: '00.00:00:00.050' + requestMethod: 'POST' + // Query text is from trusted config/queries/*.json files; no escaping needed. + requestBody: { + value: '@concat(\'{ "query": "\', pipeline().parameters.query, \' | extend x_SourceName=\\"\', pipeline().parameters.querySource, \'\\", x_SourceType=\\"\', pipeline().parameters.queryType, \'\\", x_SourceProvider=\\"\', pipeline().parameters.queryProvider, \'\\", x_SourceVersion=\\"\', pipeline().parameters.queryVersion, \'\\"" }\')' + type: 'Expression' + } + additionalHeaders: { + 'Content-Type': 'application/json' + } + } + sink: { + type: 'ParquetSink' + storeSettings: { + type: 'AzureBlobFSWriteSettings' + } + formatSettings: { + type: 'ParquetWriteSettings' + } + } + enableStaging: false + translator: { + value: '@pipeline().parameters.translator' + type: 'Expression' + } + } + inputs: [ + { + referenceName: dataset_azureResourceGraph.name + type: 'DatasetReference' + parameters: {} + } + ] + outputs: [ + { + referenceName: dataset_ingestion.name + type: 'DatasetReference' + parameters: { + blobPath: { + value: '@pipeline().parameters.ingestionPath' + type: 'Expression' + } + } + } + ] + } + ] + } + } + ] + parameters: { + query: { + type: 'String' + } + querySource: { + type: 'String' + } + queryType: { + type: 'String' + } + queryProvider: { + type: 'String' + } + queryVersion: { + type: 'String' + } + ingestionPath: { + type: 'String' + } + translator: { + type: 'Object' + } + } + policy: { + elapsedTimeMetric: {} + } + annotations: [] + } +} + + +//============================================================================== +// Outputs +//============================================================================== + +@description('The app properties for the AzureResourceGraph app.') +output app HubAppProperties = app + +@description('Metadata describing resources created by the AzureResourceGraph app.') +output metadata AzureResourceGraphMetadata = { + id: 'Microsoft.FinOpsHubs.AzureResourceGraph' + version: finOpsToolkitVersion + datasets: { + azureResourceGraph: dataset_azureResourceGraph.name + } +} diff --git a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/AzureResourceGraph/metadata.bicep b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/AzureResourceGraph/metadata.bicep new file mode 100644 index 000000000..23f321d03 --- /dev/null +++ b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/AzureResourceGraph/metadata.bicep @@ -0,0 +1,16 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +@export() +@description('Metadata for resources created by the Azure Resource Graph app.') +type AppMetadata = { + @description('Fully-qualified app identifier.') + id: string + @description('App version.') + version: string + @description('Data Factory dataset names.') + datasets: { + @description('Dataset for Azure Resource Graph REST API.') + azureResourceGraph: string + } +} diff --git a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/IngestionQueries/README.md b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/IngestionQueries/README.md new file mode 100644 index 000000000..6f3009f3b --- /dev/null +++ b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/IngestionQueries/README.md @@ -0,0 +1,128 @@ +# IngestionQueries app + +Config-driven data ingestion via query orchestration for FinOps hubs. Reads query definition files from blob storage, dispatches each to the appropriate query engine pipeline, manages file deduplication, and creates ingestion manifests for ADX. + +## How it works + +1. A daily schedule trigger fires `queries_ExecuteETL` +2. The pipeline reads all `*.json` files from `config/queries/` +3. For each query file, it calls `queries_ETL_ingestion` which: + - Normalizes the query scope (handles "Tenant" and leading slashes) + - Computes the ingestion path: `{dataset}/{scope}/{queryType}/{ingestionId}__` + - Deletes old parquet files from previous runs (same folder, different ingestion ID) + - Loads the schema mapping from `config/schemas/{dataset}_{version}.json` + - Dispatches to `queries_{queryEngine}_ExecuteQuery` via ADF REST API + - Polls for completion + - Creates a manifest file to trigger ADX ingestion + +## Adding a new query engine + +A query engine is a separate hub app that provides a single ADF pipeline capable of executing a query and writing results as Parquet. + +### Pipeline contract + +The pipeline must be named `queries_{engineName}_ExecuteQuery` and accept these parameters: + +| Parameter | Type | Description | +| --------------- | ------ | --------------------------------------------------------- | +| `query` | String | The query text to execute | +| `querySource` | String | Human-readable source name (e.g., "Azure Advisor") | +| `queryType` | String | Query type identifier (e.g., "Microsoft-AdvisorCost") | +| `queryProvider` | String | Provider identifier (e.g., "Microsoft") | +| `queryVersion` | String | Query version (e.g., "1.0") | +| `ingestionPath` | String | Full blob path prefix for the output parquet file | +| `translator` | Object | ADF TabularTranslator mapping object from the schema file | + +The pipeline must: + +- Execute the query against its data source +- Write the result as a single Parquet file to the `ingestion` container at the path specified by `ingestionPath` +- Use the `translator` for column mapping +- Append source metadata columns (`x_SourceName`, `x_SourceType`, `x_SourceProvider`, `x_SourceVersion`) to each row + +### App registration + +The engine app should register with `features: ['DataFactory']` and reference shared resources from Core via `existing` declarations. + +See `../AzureResourceGraph/app.bicep` for a reference implementation. + +### Hub wiring + +In `hub.bicep`, add the engine module with `dependsOn: [core]`: + +```bicep +module myEngine 'Microsoft.FinOpsHubs/{EngineName}/app.bicep' = if (enableMyFeature) { + name: 'Microsoft.FinOpsHubs.{EngineName}' + dependsOn: [core] + params: { + app: newApp(hub, 'Microsoft.FinOpsHubs', '{EngineName}') + } +} +``` + +## Query file format + +Query files are JSON files uploaded to `config/queries/` by data source apps (e.g., Recommendations). Each file defines a single query: + +```json +{ + "dataset": "Recommendations", + "provider": "Microsoft", + "query": "advisorresources | where type == 'microsoft.advisor/recommendations' | ...", + "queryEngine": "ResourceGraph", + "scope": "Tenant", + "source": "Azure Advisor", + "type": "Microsoft-AdvisorCost", + "version": "1.0" +} +``` + +| Field | Description | +| ------------- | -------------------------------------------------------------------------------- | +| `dataset` | Target managed dataset name (determines top-level folder in ingestion container) | +| `provider` | Publisher/provider identifier | +| `query` | The query text passed to the engine pipeline | +| `queryEngine` | Engine name; must match a `queries_{queryEngine}_ExecuteQuery` pipeline | +| `scope` | Query scope; "Tenant" is normalized to `tenants/{tenantId}` at runtime | +| `source` | Human-readable source name for `x_SourceName` | +| `type` | Query type identifier; used as subfolder under scope | +| `version` | Query version; determines which schema file to load (`{dataset}_{version}.json`) | + +## Schema file format + +Schema files live in `config/schemas/` and follow the ADF TabularTranslator format: + +```json +{ + "additionalColumns": [], + "translator": { + "type": "TabularTranslator", + "mappings": [{ "source": { "path": "[['ColumnName']" }, "sink": { "path": "ColumnName" } }], + "collectionReference": "$['data']" + } +} +``` + +## Folder structure in ingestion container + +``` +ingestion/ + {dataset}/ + {scope}/ + {queryType}/ + {ingestionId}__{queryType}.parquet + manifest.json +``` + +For recommendations with scope "Tenant": + +``` +ingestion/ + Recommendations/ + tenants/{tenantId}/ + Microsoft-AdvisorCost/ + 20250219-010100_a1b2c3d4__Microsoft-AdvisorCost.parquet + manifest.json +``` + +Each run replaces the previous parquet files (same folder path, different ingestion ID prefix). diff --git a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/IngestionQueries/app.bicep b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/IngestionQueries/app.bicep new file mode 100644 index 000000000..319b04852 --- /dev/null +++ b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/IngestionQueries/app.bicep @@ -0,0 +1,803 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { finOpsToolkitVersion, HubAppProperties, isSupportedVersion } from '../../fx/hub-types.bicep' +import { AppMetadata as CoreMetadata } from '../Core/metadata.bicep' +import { AppMetadata as IngestionQueriesMetadata } from './metadata.bicep' + +metadata hubApp = { + id: 'Microsoft.FinOpsHubs.IngestionQueries' + version: '$$ftkver$$' + dependencies: ['Microsoft.FinOpsHubs.Core'] + metadata: 'https://microsoft.github.io/finops-toolkit/deploy/$$ftkver$$/Microsoft.FinOpsHubs/IngestionQueries/metadata.bicep' +} + + +//============================================================================== +// Parameters +//============================================================================== + +@description('Required. FinOps hub app getting deployed.') +param app HubAppProperties + +@description('Required. Metadata describing shared resources from the Core app. Must be v13 or higher.') +@validate(x => isSupportedVersion(x.version, '13.0', ''), 'Core app version must be 13.0 or higher.') +param core CoreMetadata + + +//============================================================================== +// Variables +//============================================================================== + +var QUERIES = 'queries' + + +//============================================================================== +// Resources +//============================================================================== + +// Register app +module appRegistration '../../fx/hub-app.bicep' = { + name: 'Microsoft.FinOpsHubs.IngestionQueries_Register' + params: { + app: app + version: finOpsToolkitVersion + features: [ + 'DataFactory' + ] + } +} + +// Get ADF resources +resource dataFactory 'Microsoft.DataFactory/factories@2018-06-01' existing = { + name: app.dataFactory + dependsOn: [appRegistration] + + resource dataset_config 'datasets@2018-06-01' existing = { + name: core.datasets.config + } + resource dataset_ingestion 'datasets@2018-06-01' existing = { + name: core.datasets.ingestion + } + resource dataset_ingestion_files 'datasets@2018-06-01' existing = { + name: core.datasets.ingestionFiles + } + resource dataset_manifest 'datasets@2018-06-01' existing = { + name: core.datasets.ingestionManifest + } +} + +//------------------------------------------------------------------------------ +// Scheduling +//------------------------------------------------------------------------------ + +module timeZones '../../Microsoft.CostManagement/ManagedExports/timeZones.bicep' = { + name: 'Microsoft.FinOpsHubs.IngestionQueries_TimeZones' + params: { + location: app.hub.location + } +} + +resource trigger_DailySchedule 'Microsoft.DataFactory/factories/triggers@2018-06-01' = { + name: '${QUERIES}_DailySchedule' + parent: dataFactory + properties: { + pipelines: [ + { + pipelineReference: { + referenceName: pipeline_ExecuteQueries.name + type: 'PipelineReference' + } + parameters: {} + } + ] + type: 'ScheduleTrigger' + typeProperties: { + recurrence: { + frequency: 'Hour' + interval: 24 + startTime: '2023-01-01T01:01:00' + timeZone: timeZones.outputs.Timezone + } + } + } +} + +//------------------------------------------------------------------------------ +// Pipelines +//------------------------------------------------------------------------------ + +resource pipeline_ExecuteQueries 'Microsoft.DataFactory/factories/pipelines@2018-06-01' = { + name: '${QUERIES}_ExecuteETL' + parent: dataFactory + properties: { + activities: [ + { // Load Queries + name: 'Load Queries' + type: 'Lookup' + dependsOn: [] + policy: { + timeout: '0.00:10:00' + retry: 0 + retryIntervalInSeconds: 30 + secureOutput: false + secureInput: false + } + userProperties: [] + typeProperties: { + source: { + type: 'JsonSource' + storeSettings: { + type: 'AzureBlobFSReadSettings' + recursive: true + wildcardFileName: '*.json' + enablePartitionDiscovery: false + } + formatSettings: { + type: 'JsonReadSettings' + } + } + dataset: { + referenceName: dataFactory::dataset_config.name + type: 'DatasetReference' + parameters: { + fileName: '*.json' + folderPath: '${core.containers.config}/${QUERIES}' + } + } + firstRowOnly: false + } + } + { // Set Ingestion Id + name: 'Set Ingestion Id' + type: 'SetVariable' + dependsOn: [] + policy: { + secureOutput: false + secureInput: false + } + userProperties: [] + typeProperties: { + variableName: 'ingestionId' + value: { + value: '@concat(utcNow(\'yyyyMMdd-HHmmss\'), \'_\', substring(guid(), 0, 8))' + type: 'Expression' + } + } + } + { // Loop Thru Queries + name: 'Loop Thru Queries' + type: 'ForEach' + dependsOn: [ + { + activity: 'Load Queries' + dependencyConditions: ['Succeeded'] + } + { + activity: 'Set Ingestion Id' + dependencyConditions: ['Succeeded'] + } + ] + userProperties: [] + typeProperties: { + items: { + value: '@activity(\'Load Queries\').output.value' + type: 'Expression' + } + batchCount: 2 + isSequential: false + activities: [ + { // Execute File Queries + name: 'Execute File Queries' + description: 'Execute the queries declared in the queries file.' + type: 'ExecutePipeline' + dependsOn: [] + policy: { + secureInput: false + } + userProperties: [] + typeProperties: { + pipeline: { + referenceName: pipeline_ExecuteQueries_query.name + type: 'PipelineReference' + } + waitOnCompletion: true + parameters: { + ingestionId: { + value: '@variables(\'ingestionId\')' + type: 'Expression' + } + queryEngine: { + value: '@item().queryEngine' + type: 'Expression' + } + outputDataset: { + value: '@item().dataset' + type: 'Expression' + } + schemaFile: { + value: '@concat(toLower(item().dataset), \'_\', item().version, \'.json\')' + type: 'Expression' + } + queryScope: { + value: '@item().scope' + type: 'Expression' + } + query: { + value: '@item().query' + type: 'Expression' + } + queryVersion: { + value: '@item().version' + type: 'Expression' + } + querySource: { + value: '@item().source' + type: 'Expression' + } + queryProvider: { + value: '@item().provider' + type: 'Expression' + } + queryType: { + value: '@item().type' + type: 'Expression' + } + } + } + } + ] + } + } + ] + parameters: {} + variables: { + ingestionId: { + type: 'String' + } + } + policy: { + elapsedTimeMetric: {} + } + annotations: [] + } +} + +resource pipeline_ExecuteQueries_query 'Microsoft.DataFactory/factories/pipelines@2018-06-01' = { + name: '${QUERIES}_ETL_${core.containers.ingestion}' + parent: dataFactory + properties: { + activities: [ + { // Normalize Query Scope + name: 'Normalize Query Scope' + description: 'Replace "Tenant" with "tenants/{tenantId}" to ensure unique paths across Remote Hubs.' + type: 'SetVariable' + dependsOn: [] + policy: { + secureOutput: false + secureInput: false + } + userProperties: [] + typeProperties: { + variableName: 'queryScope' + value: { + value: '@if(equals(toLower(pipeline().parameters.queryScope), \'tenant\'), \'tenants/${tenant().tenantId}\', if(startsWith(pipeline().parameters.queryScope, \'/\'), substring(pipeline().parameters.queryScope, 1), pipeline().parameters.queryScope))' + type: 'Expression' + } + } + } + { // Set Ingestion Path + name: 'Set Ingestion Path' + type: 'SetVariable' + dependsOn: [ + { + activity: 'Normalize Query Scope' + dependencyConditions: ['Succeeded'] + } + ] + policy: { + secureOutput: false + secureInput: false + } + userProperties: [] + typeProperties: { + variableName: 'ingestionPath' + value: { + value: '@concat(pipeline().parameters.outputDataset, \'/\', variables(\'queryScope\'), \'/\', pipeline().parameters.queryType, \'/\', pipeline().parameters.ingestionId, \'${core.ingestionIdFileNameSeparator}\')' + type: 'Expression' + } + } + } + { // Get Existing Parquet Files + name: 'Get Existing Parquet Files' + description: 'Get the previously ingested files so we can remove any older data.' + type: 'GetMetadata' + dependsOn: [ + { + activity: 'Normalize Query Scope' + dependencyConditions: ['Succeeded'] + } + ] + policy: { + timeout: '0.12:00:00' + retry: 0 + retryIntervalInSeconds: 30 + secureOutput: false + secureInput: false + } + userProperties: [] + typeProperties: { + dataset: { + referenceName: dataFactory::dataset_ingestion_files.name + type: 'DatasetReference' + parameters: { + folderPath: '@concat(pipeline().parameters.outputDataset, \'/\', variables(\'queryScope\'), \'/\', pipeline().parameters.queryType)' + } + } + fieldList: [ + 'exists' + 'childItems' + ] + storeSettings: { + type: 'AzureBlobFSReadSettings' + enablePartitionDiscovery: false + } + formatSettings: { + type: 'ParquetReadSettings' + } + } + } + { // Filter Out Current Exports + name: 'Filter Out Current Exports' + description: 'Remove existing files from the current export so those files do not get deleted.' + type: 'Filter' + dependsOn: [ + { + activity: 'Get Existing Parquet Files' + dependencyConditions: ['Succeeded'] + } + ] + userProperties: [] + typeProperties: { + items: { + value: '@if(and(activity(\'Get Existing Parquet Files\').output.exists, contains(activity(\'Get Existing Parquet Files\').output, \'childItems\')), activity(\'Get Existing Parquet Files\').output.childItems, json(\'[]\'))' + type: 'Expression' + } + condition: { + value: '@not(startswith(item().name, concat(pipeline().parameters.ingestionId, \'${core.ingestionIdFileNameSeparator}\')))' + type: 'Expression' + } + } + } + { // Delete Old Files Loop + name: 'Delete Old Files Loop' + description: 'Loop thru each of the existing files from previous exports.' + type: 'ForEach' + dependsOn: [ + { + activity: 'Filter Out Current Exports' + dependencyConditions: ['Succeeded'] + } + ] + userProperties: [] + typeProperties: { + items: { + value: '@activity(\'Filter Out Current Exports\').output.Value' + type: 'Expression' + } + activities: [ + { // Delete Old Ingested File + name: 'Delete Old Ingested File' + description: 'Delete the previously ingested files from older exports.' + type: 'Delete' + dependsOn: [] + policy: { + timeout: '0.12:00:00' + retry: 0 + retryIntervalInSeconds: 30 + secureOutput: false + secureInput: false + } + userProperties: [] + typeProperties: { + dataset: { + referenceName: dataFactory::dataset_ingestion.name + type: 'DatasetReference' + parameters: { + blobPath: { + value: '@concat(pipeline().parameters.outputDataset, \'/\', variables(\'queryScope\'), \'/\', pipeline().parameters.queryType, \'/\', item().name)' + type: 'Expression' + } + } + } + enableLogging: false + storeSettings: { + type: 'AzureBlobFSReadSettings' + recursive: false + enablePartitionDiscovery: false + } + } + } + ] + } + } + { // Load Schema Mappings + name: 'Load Schema Mappings' + type: 'Lookup' + dependsOn: [] + policy: { + timeout: '0.12:00:00' + retry: 0 + retryIntervalInSeconds: 30 + secureOutput: false + secureInput: false + } + userProperties: [] + typeProperties: { + source: { + type: 'JsonSource' + storeSettings: { + type: 'AzureBlobFSReadSettings' + recursive: true + enablePartitionDiscovery: false + } + formatSettings: { + type: 'JsonReadSettings' + } + } + dataset: { + referenceName: dataFactory::dataset_config.name + type: 'DatasetReference' + parameters: { + fileName: { + value: '@pipeline().parameters.schemaFile' + type: 'Expression' + } + folderPath: '${core.containers.config}/schemas' + } + } + } + } + { // Run Query Engine Pipeline + name: 'Run Query Engine Pipeline' + description: 'Dynamically dispatch to the engine-specific Copy pipeline via ADF REST API.' + type: 'WebActivity' + dependsOn: [ + { + activity: 'Set Ingestion Path' + dependencyConditions: ['Succeeded'] + } + { + activity: 'Delete Old Files Loop' + dependencyConditions: ['Succeeded'] + } + { + activity: 'Load Schema Mappings' + dependencyConditions: ['Succeeded'] + } + ] + policy: { + timeout: '0.00:10:00' + retry: 0 + retryIntervalInSeconds: 30 + secureOutput: false + secureInput: false + } + userProperties: [] + typeProperties: { + url: { + value: '@concat(\'${environment().resourceManager}subscriptions/${subscription().subscriptionId}/resourceGroups/${resourceGroup().name}/providers/Microsoft.DataFactory/factories/\', pipeline().DataFactory, \'/pipelines/queries_\', pipeline().parameters.queryEngine, \'_ExecuteQuery/createRun?api-version=2018-06-01\')' + type: 'Expression' + } + method: 'POST' + headers: { + 'Content-Type': 'application/json' + } + body: { + value: '@json(concat(\'{"query":"\', pipeline().parameters.query, \'","querySource":"\', pipeline().parameters.querySource, \'","queryType":"\', pipeline().parameters.queryType, \'","queryProvider":"\', pipeline().parameters.queryProvider, \'","queryVersion":"\', pipeline().parameters.queryVersion, \'","ingestionPath":"\', concat(variables(\'ingestionPath\'), pipeline().parameters.queryType, \'.parquet\'), \'","translator":\', string(activity(\'Load Schema Mappings\').output.firstRow.translator), \'}\'))' + type: 'Expression' + } + authentication: { + type: 'MSI' + resource: environment().resourceManager + } + } + } + { // Wait For Query Engine Pipeline + name: 'Wait For Query Engine Pipeline' + description: 'Poll for engine pipeline completion using the runId from createRun.' + type: 'Until' + dependsOn: [ + { + activity: 'Run Query Engine Pipeline' + dependencyConditions: ['Succeeded'] + } + ] + userProperties: [] + typeProperties: { + expression: { + value: '@or(equals(variables(\'engineRunStatus\'), \'Succeeded\'), equals(variables(\'engineRunStatus\'), \'Failed\'), equals(variables(\'engineRunStatus\'), \'Cancelled\'))' + type: 'Expression' + } + activities: [ + { // Check Query Engine Pipeline Status + name: 'Check Query Engine Pipeline Status' + type: 'WebActivity' + dependsOn: [] + policy: { + timeout: '0.00:05:00' + retry: 2 + retryIntervalInSeconds: 30 + secureOutput: false + secureInput: false + } + userProperties: [] + typeProperties: { + url: { + value: '@concat(\'${environment().resourceManager}subscriptions/${subscription().subscriptionId}/resourceGroups/${resourceGroup().name}/providers/Microsoft.DataFactory/factories/\', pipeline().DataFactory, \'/pipelineruns/\', activity(\'Run Query Engine Pipeline\').output.runId, \'?api-version=2018-06-01\')' + type: 'Expression' + } + method: 'GET' + authentication: { + type: 'MSI' + resource: environment().resourceManager + } + } + } + { // Set Engine Run Status + name: 'Set Engine Run Status' + type: 'SetVariable' + dependsOn: [ + { + activity: 'Check Query Engine Pipeline Status' + dependencyConditions: ['Succeeded'] + } + ] + policy: { + secureOutput: false + secureInput: false + } + userProperties: [] + typeProperties: { + variableName: 'engineRunStatus' + value: { + value: '@activity(\'Check Query Engine Pipeline Status\').output.status' + type: 'Expression' + } + } + } + { // Wait Before Retry + name: 'Wait Before Retry' + type: 'Wait' + dependsOn: [ + { + activity: 'Set Engine Run Status' + dependencyConditions: ['Succeeded'] + } + ] + userProperties: [] + typeProperties: { + waitTimeInSeconds: 15 + } + } + ] + timeout: '0.00:30:00' + } + } + { // Verify Query Engine Pipeline Succeeded + name: 'Verify Query Engine Pipeline Succeeded' + description: 'Fail the pipeline if the query engine run did not succeed.' + type: 'IfCondition' + dependsOn: [ + { + activity: 'Wait For Query Engine Pipeline' + dependencyConditions: ['Succeeded'] + } + ] + userProperties: [] + typeProperties: { + expression: { + value: '@not(equals(variables(\'engineRunStatus\'), \'Succeeded\'))' + type: 'Expression' + } + ifTrueActivities: [ + { + name: 'Query Engine Pipeline Failed' + type: 'Fail' + dependsOn: [] + userProperties: [] + typeProperties: { + message: { + value: '@concat(\'Query engine pipeline finished with status: \', variables(\'engineRunStatus\'))' + type: 'Expression' + } + errorCode: 'QueryEnginePipelineFailed' + } + } + ] + } + } + { // Check If Data Was Written + name: 'Check If Data Was Written' + description: 'Check if the query engine actually wrote output files before creating a manifest.' + type: 'GetMetadata' + dependsOn: [ + { + activity: 'Verify Query Engine Pipeline Succeeded' + dependencyConditions: ['Succeeded'] + } + ] + policy: { + timeout: '0.12:00:00' + retry: 0 + retryIntervalInSeconds: 30 + secureOutput: false + secureInput: false + } + userProperties: [] + typeProperties: { + dataset: { + referenceName: dataFactory::dataset_ingestion_files.name + type: 'DatasetReference' + parameters: { + folderPath: '@concat(pipeline().parameters.outputDataset, \'/\', variables(\'queryScope\'), \'/\', pipeline().parameters.queryType)' + } + } + fieldList: ['exists'] + storeSettings: { + type: 'AzureBlobFSReadSettings' + enablePartitionDiscovery: false + } + formatSettings: { + type: 'ParquetReadSettings' + } + } + } + { // Create Manifest If Data Exists + name: 'Create Manifest If Data Exists' + description: 'Only create a manifest file when query results were written, to avoid triggering ADX ingestion on empty folders.' + type: 'IfCondition' + dependsOn: [ + { + activity: 'Check If Data Was Written' + dependencyConditions: ['Succeeded'] + } + ] + userProperties: [] + typeProperties: { + expression: { + value: '@activity(\'Check If Data Was Written\').output.exists' + type: 'Expression' + } + ifTrueActivities: [ + { // Create Manifest + name: 'Create Manifest' + description: 'Copy the settings file as manifest.json to the ingestion folder to trigger ADX ingestion.' + type: 'Copy' + dependsOn: [] + policy: { + timeout: '0.12:00:00' + retry: 0 + retryIntervalInSeconds: 30 + secureInput: false + secureOutput: false + } + userProperties: [] + typeProperties: { + source: { + type: 'JsonSource' + storeSettings: { + type: 'AzureBlobFSReadSettings' + recursive: true + enablePartitionDiscovery: false + } + formatSettings: { + type: 'JsonReadSettings' + } + } + sink: { + type: 'JsonSink' + storeSettings: { + type: 'AzureBlobFSWriteSettings' + } + formatSettings: { + type: 'JsonWriteSettings' + } + } + enableStaging: false + } + inputs: [ + { + referenceName: dataFactory::dataset_config.name + type: 'DatasetReference' + parameters: { + fileName: core.settings.file + folderPath: core.settings.container + } + } + ] + outputs: [ + { + referenceName: dataFactory::dataset_manifest.name + type: 'DatasetReference' + parameters: { + fileName: 'manifest.json' + folderPath: { + value: '@concat(\'${core.containers.ingestion}/\', pipeline().parameters.outputDataset, \'/\', variables(\'queryScope\'), \'/\', pipeline().parameters.queryType)' + type: 'Expression' + } + } + } + ] + } + ] + } + } + ] + parameters: { + ingestionId: { + type: 'String' + } + queryEngine: { + type: 'String' + } + outputDataset: { + type: 'String' + } + schemaFile: { + type: 'String' + } + queryScope: { + type: 'String' + } + query: { + type: 'String' + } + queryVersion: { + type: 'String' + } + querySource: { + type: 'String' + } + queryProvider: { + type: 'String' + } + queryType: { + type: 'String' + } + } + variables: { + queryScope: { + type: 'String' + } + ingestionPath: { + type: 'String' + } + engineRunStatus: { + type: 'String' + } + } + policy: { + elapsedTimeMetric: {} + } + annotations: [] + } +} + + +//============================================================================== +// Outputs +//============================================================================== + +@description('The app properties for the IngestionQueries app.') +output app HubAppProperties = app + +@description('Metadata describing resources created by the IngestionQueries app.') +output metadata IngestionQueriesMetadata = { + id: 'Microsoft.FinOpsHubs.IngestionQueries' + version: finOpsToolkitVersion + queries: { + container: core.containers.config + folder: QUERIES + } +} diff --git a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/IngestionQueries/metadata.bicep b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/IngestionQueries/metadata.bicep new file mode 100644 index 000000000..5e39e8825 --- /dev/null +++ b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/IngestionQueries/metadata.bicep @@ -0,0 +1,22 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//============================================================================== +// App metadata definition +//============================================================================== + +@export() +@description('Metadata for resources created by the Ingestion Queries app.') +type AppMetadata = { + @description('Fully-qualified app identifier.') + id: string + @description('App version.') + version: string + @description('Metadata for query definition files.') + queries: { + @description('Container name for query definition files.') + container: string + @description('Folder path for query definition files within the container.') + folder: string + } +} diff --git a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Recommendations/app.bicep b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Recommendations/app.bicep new file mode 100644 index 000000000..d8db204ea --- /dev/null +++ b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Recommendations/app.bicep @@ -0,0 +1,106 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { finOpsToolkitVersion, HubAppProperties, isSupportedVersion } from '../../fx/hub-types.bicep' +import { AppMetadata as CoreMetadata } from '../Core/metadata.bicep' +import { AppMetadata as IngestionQueriesMetadata } from '../IngestionQueries/metadata.bicep' + +metadata hubApp = { + id: 'Microsoft.FinOpsHubs.Recommendations' + version: '$$ftkver$$' + dependencies: [ + 'Microsoft.FinOpsHubs.Core' + 'Microsoft.FinOpsHubs.IngestionQueries' + 'Microsoft.FinOpsHubs.AzureResourceGraph' + ] + metadata: 'https://microsoft.github.io/finops-toolkit/deploy/$$ftkver$$/Microsoft.FinOpsHubs/Recommendations/metadata.bicep' +} + + +//============================================================================== +// Parameters +//============================================================================== + +@description('Required. FinOps hub app getting deployed.') +param app HubAppProperties + +@description('Optional. Indicates whether to enable Azure Hybrid Benefit recommendations. These recommendations flag VMs and SQL VMs without Azure Hybrid Benefit enabled, which may generate noise if your organization does not have on-premises licenses. Default: false.') +param enableAHBRecommendations bool = false + +@description('Optional. Indicates whether to enable non-Spot AKS cluster recommendations. These recommendations flag AKS clusters that use autoscaling without Spot VMs, which may generate noise since Spot VMs are only appropriate for interruptible workloads. Default: false.') +param enableSpotRecommendations bool = false + +@description('Required. Metadata describing shared resources from the Core app. Must be v13 or higher.') +@validate(x => isSupportedVersion(x.version, '13.0', ''), 'Core app version must be 13.0 or higher.') +param core CoreMetadata + +@description('Required. Metadata describing resources from the Ingestion Queries app. Must be v13 or higher.') +@validate(x => isSupportedVersion(x.version, '13.0', ''), 'IngestionQueries app version must be 13.0 or higher.') +param ingestionQueries IngestionQueriesMetadata + +//============================================================================== +// Variables +//============================================================================== + +// +// Query file entries are generated during build by Build-HubIngestionQueries.ps1. +// Do not edit this section manually. The build script scans the queries/ folder and +// generates loadTextContent entries grouped by the optional "group" field in each JSON file. +var queryFiles = {} +// + +// Load schema files +var schemaFiles = { + 'recommendations_1.0': loadTextContent('schemas/recommendations_1.0.json') +} + + +//============================================================================== +// Resources +//============================================================================== + +// Register app +module appRegistration '../../fx/hub-app.bicep' = { + name: 'Microsoft.FinOpsHubs.Recommendations_Register' + params: { + app: app + version: finOpsToolkitVersion + features: [ + 'Storage' // Storing queries and schemas + ] + } +} + +//------------------------------------------------------------------------------ +// Storage +//------------------------------------------------------------------------------ + +// Upload query files to storage +module uploadQueries '../../fx/hub-storage.bicep' = { + name: 'Microsoft.FinOpsHubs.Recommendations_UploadQueries' + dependsOn: [appRegistration] + params: { + app: app + container: ingestionQueries.queries.container + files: reduce(items(queryFiles), {}, (acc, item) => union(acc, { '${ingestionQueries.queries.folder}/${item.key}.json': item.value })) + } +} + +// Upload schema files to storage +module uploadSchemas '../../fx/hub-storage.bicep' = { + name: 'Microsoft.FinOpsHubs.Recommendations_UploadSchemas' + dependsOn: [appRegistration] + params: { + app: app + container: core.containers.config + files: reduce(items(schemaFiles), {}, (acc, item) => union(acc, { 'schemas/${item.key}.json': item.value })) + } +} + + +//============================================================================== +// Outputs +//============================================================================== + +@description('The app properties for the Recommendations app.') +output app HubAppProperties = app diff --git a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Recommendations/queries/Recommendations-Microsoft-AdvisorCost.json b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Recommendations/queries/Recommendations-Microsoft-AdvisorCost.json new file mode 100644 index 000000000..66b0c5d1c --- /dev/null +++ b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Recommendations/queries/Recommendations-Microsoft-AdvisorCost.json @@ -0,0 +1,10 @@ +{ + "dataset": "Recommendations", + "provider": "Microsoft", + "query": "advisorresources | where type == 'microsoft.advisor/recommendations' | where properties.category == 'Cost' | extend x_RecommendationDetails = bag_pack('RecommendationImpact', tostring(properties.impact), 'x_RecommendationProvider', 'Azure Advisor', 'x_RecommendationSolution', tostring(properties.shortDescription.solution), 'x_RecommendationTypeId', tostring(properties.recommendationTypeId), 'x_ResourceType', tolower(properties.impactedField)) | extend x_RecommendationDetails = tostring(bag_merge(x_RecommendationDetails, properties.extendedProperties)) | project x_RecommendationId=id, x_ResourceGroupName=tolower(resourceGroup), SubAccountId=subscriptionId, x_RecommendationCategory=tostring(properties.category), x_RecommendationDescription=tostring(properties.shortDescription.problem), ResourceId=tolower(properties.resourceMetadata.resourceId), ResourceName=tolower(properties.impactedValue), x_RecommendationDetails, x_RecommendationDate=tostring(properties.lastUpdated) | join kind=leftouter ( resourcecontainers | where type == 'microsoft.resources/subscriptions' | project SubAccountName=name, SubAccountId=subscriptionId ) on SubAccountId | project-away SubAccountId1", + "queryEngine": "ResourceGraph", + "scope": "Tenant", + "source": "Azure Advisor", + "type": "Microsoft-AdvisorCost", + "version": "1.0" +} diff --git a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Recommendations/queries/Recommendations-Microsoft-BackendlessAppGateways.json b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Recommendations/queries/Recommendations-Microsoft-BackendlessAppGateways.json new file mode 100644 index 000000000..31c5d2b6e --- /dev/null +++ b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Recommendations/queries/Recommendations-Microsoft-BackendlessAppGateways.json @@ -0,0 +1,10 @@ +{ + "dataset": "Recommendations", + "provider": "Microsoft", + "query": "resources | where type =~ 'Microsoft.Network/applicationGateways' | extend backendPoolsCount = array_length(properties.backendAddressPools),SKUName= tostring(properties.sku.name), SKUTier= tostring(properties.sku.tier),SKUCapacity=properties.sku.capacity,backendPools=properties.backendAddressPools| project id, name, SKUName, SKUTier, SKUCapacity,resourceGroup,subscriptionId, AppGWName=name, type, Location=location| join ( resources | where type =~ 'Microsoft.Network/applicationGateways' | mvexpand backendPools = properties.backendAddressPools | extend backendIPCount = array_length(backendPools.properties.backendIPConfigurations) | extend backendAddressesCount = array_length(backendPools.properties.backendAddresses) | extend backendPoolName = backendPools.properties.backendAddressPools.name | summarize backendIPCount = sum(backendIPCount) ,backendAddressesCount=sum(backendAddressesCount) by id) on id| project-away id1| where (backendIPCount == 0 or isempty(backendIPCount)) and (backendAddressesCount==0 or isempty(backendAddressesCount))| project x_RecommendationId=strcat(tolower(id),'-idle'), x_ResourceGroupName=tolower(resourceGroup), SubAccountId=subscriptionId, x_RecommendationCategory='Cost', x_RecommendationDescription='Application Gateway without any backend pool', ResourceId = tolower(id), ResourceName=tolower(AppGWName), x_RecommendationDetails= tostring(bag_pack('backendIPCount', backendIPCount, 'Location', Location, 'x_RecommendationProvider', 'FinOps hubs', 'x_RecommendationSolution', 'Review and remove this resource if not needed', 'x_RecommendationTypeId', '4f69df93-5972-44e0-97cf-4343c2bcf4b8', 'x_ResourceType', type, 'x_RecommendationMaturityLevel', 'Preview')), x_RecommendationDate = now() | join kind=leftouter ( resourcecontainers | where type == 'microsoft.resources/subscriptions' | project SubAccountName=name, SubAccountId=subscriptionId ) on SubAccountId | project-away SubAccountId1", + "queryEngine": "ResourceGraph", + "scope": "Tenant", + "source": "FinOps hubs", + "type": "Microsoft-BackendlessAppGateways", + "version": "1.0" +} diff --git a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Recommendations/queries/Recommendations-Microsoft-BackendlessLoadBalancers.json b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Recommendations/queries/Recommendations-Microsoft-BackendlessLoadBalancers.json new file mode 100644 index 000000000..0ca3f1b14 --- /dev/null +++ b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Recommendations/queries/Recommendations-Microsoft-BackendlessLoadBalancers.json @@ -0,0 +1,10 @@ +{ + "dataset": "Recommendations", + "provider": "Microsoft", + "query": "resources | where type =~ 'microsoft.network/loadbalancers' and array_length(properties.backendAddressPools) == 0 and sku.name!='Basic' | extend SKUName=tostring(sku.name) | extend SKUTier=tostring(sku.tier), Location=location | extend backendAddressPools = properties.backendAddressPools | extend id,name, SKUName,SKUTier,backendAddressPools, location,resourceGroup, subscriptionId, type| project x_RecommendationId=strcat(tolower(id),'-idle'), x_ResourceGroupName=tolower(resourceGroup), SubAccountId=subscriptionId, x_RecommendationCategory='Cost', x_RecommendationDescription='Load balancer without a backend pool', ResourceId = tolower(id), ResourceName=tolower(name), x_RecommendationDetails= tostring(bag_pack('SKUName', SKUName, 'SKUTier', SKUTier, 'Location', Location, 'x_RecommendationProvider', 'FinOps hubs', 'x_RecommendationSolution', 'Review and remove this resource if not needed', 'x_RecommendationTypeId', 'ab703887-fa23-4915-abdc-3defbea89f7a', 'x_ResourceType', type)), x_RecommendationDate = now() | join kind=leftouter ( resourcecontainers | where type == 'microsoft.resources/subscriptions' | project SubAccountName=name, SubAccountId=subscriptionId ) on SubAccountId | project-away SubAccountId1", + "queryEngine": "ResourceGraph", + "scope": "Tenant", + "source": "FinOps hubs", + "type": "Microsoft-BackendlessLoadBalancers", + "version": "1.0" +} diff --git a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Recommendations/queries/Recommendations-Microsoft-BasicLoadBalancers.json b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Recommendations/queries/Recommendations-Microsoft-BasicLoadBalancers.json new file mode 100644 index 000000000..7d0c667ae --- /dev/null +++ b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Recommendations/queries/Recommendations-Microsoft-BasicLoadBalancers.json @@ -0,0 +1,10 @@ +{ + "dataset": "Recommendations", + "provider": "Microsoft", + "query": "resources | where type =~ 'microsoft.network/loadbalancers' | where sku.name =~ 'Basic' | extend Location=location | project id, name, resourceGroup, subscriptionId, Location, type | project x_RecommendationId=strcat(tolower(id),'-basicLoadBalancer'), x_ResourceGroupName=tolower(resourceGroup), SubAccountId=subscriptionId, x_RecommendationCategory='Cost', x_RecommendationDescription='Load balancer is using the retired Basic SKU', ResourceId = tolower(id), ResourceName=tolower(name), x_RecommendationDetails = tostring(bag_pack('Location', Location, 'x_RecommendationProvider', 'FinOps hubs', 'x_RecommendationSolution', 'Upgrade to Standard SKU load balancer', 'x_RecommendationTypeId', 'a2b3c4d5-e6f7-8a9b-0c1d-2e3f4a5b6c7d', 'x_ResourceType', type)), x_RecommendationDate = now() | join kind=leftouter ( resourcecontainers | where type == 'microsoft.resources/subscriptions' | project SubAccountName=name, SubAccountId=subscriptionId ) on SubAccountId | project-away SubAccountId1", + "queryEngine": "ResourceGraph", + "scope": "Tenant", + "source": "FinOps hubs", + "type": "Microsoft-BasicLoadBalancers", + "version": "1.0" +} diff --git a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Recommendations/queries/Recommendations-Microsoft-BasicPublicIPs.json b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Recommendations/queries/Recommendations-Microsoft-BasicPublicIPs.json new file mode 100644 index 000000000..677a4474e --- /dev/null +++ b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Recommendations/queries/Recommendations-Microsoft-BasicPublicIPs.json @@ -0,0 +1,10 @@ +{ + "dataset": "Recommendations", + "provider": "Microsoft", + "query": "resources | where type =~ 'microsoft.network/publicipaddresses' | where sku.name =~ 'Basic' | extend Location=location, AllocationMethod=tostring(properties.publicIPAllocationMethod), IpAddress=tostring(properties.ipAddress) | project id, name, resourceGroup, subscriptionId, Location, AllocationMethod, IpAddress, type | project x_RecommendationId=strcat(tolower(id),'-basicPublicIP'), x_ResourceGroupName=tolower(resourceGroup), SubAccountId=subscriptionId, x_RecommendationCategory='Cost', x_RecommendationDescription='Public IP is using the retired Basic SKU', ResourceId = tolower(id), ResourceName=tolower(name), x_RecommendationDetails = tostring(bag_pack('AllocationMethod', AllocationMethod, 'IpAddress', IpAddress, 'Location', Location, 'x_RecommendationProvider', 'FinOps hubs', 'x_RecommendationSolution', 'Upgrade to Standard SKU public IP', 'x_RecommendationTypeId', 'f1a2b3c4-d5e6-7f8a-9b0c-1d2e3f4a5b6c', 'x_ResourceType', type)), x_RecommendationDate = now() | join kind=leftouter ( resourcecontainers | where type == 'microsoft.resources/subscriptions' | project SubAccountName=name, SubAccountId=subscriptionId ) on SubAccountId | project-away SubAccountId1", + "queryEngine": "ResourceGraph", + "scope": "Tenant", + "source": "FinOps hubs", + "type": "Microsoft-BasicPublicIPs", + "version": "1.0" +} diff --git a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Recommendations/queries/Recommendations-Microsoft-ClassicAppGateways.json b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Recommendations/queries/Recommendations-Microsoft-ClassicAppGateways.json new file mode 100644 index 000000000..55acbae59 --- /dev/null +++ b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Recommendations/queries/Recommendations-Microsoft-ClassicAppGateways.json @@ -0,0 +1,10 @@ +{ + "dataset": "Recommendations", + "provider": "Microsoft", + "query": "resources | where type =~ 'microsoft.network/applicationgateways' | where tostring(properties.sku.tier) in~ ('Standard', 'WAF') | extend Location=location, SKUName=tostring(properties.sku.name), SKUTier=tostring(properties.sku.tier), SKUCapacity=tostring(properties.sku.capacity) | project id, name, resourceGroup, subscriptionId, Location, SKUName, SKUTier, SKUCapacity, type | project x_RecommendationId=strcat(tolower(id),'-classicAppGateway'), x_ResourceGroupName=tolower(resourceGroup), SubAccountId=subscriptionId, x_RecommendationCategory='Cost', x_RecommendationDescription='Application Gateway is using the retiring v1 SKU', ResourceId = tolower(id), ResourceName=tolower(name), x_RecommendationDetails = tostring(bag_pack('SKUName', SKUName, 'SKUTier', SKUTier, 'SKUCapacity', SKUCapacity, 'Location', Location, 'x_RecommendationProvider', 'FinOps hubs', 'x_RecommendationSolution', 'Migrate to Application Gateway v2 (Standard_v2 or WAF_v2)', 'x_RecommendationTypeId', 'b3c4d5e6-f7a8-9b0c-1d2e-3f4a5b6c7d8e', 'x_ResourceType', type)), x_RecommendationDate = now() | join kind=leftouter ( resourcecontainers | where type == 'microsoft.resources/subscriptions' | project SubAccountName=name, SubAccountId=subscriptionId ) on SubAccountId | project-away SubAccountId1", + "queryEngine": "ResourceGraph", + "scope": "Tenant", + "source": "FinOps hubs", + "type": "Microsoft-ClassicAppGateways", + "version": "1.0" +} diff --git a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Recommendations/queries/Recommendations-Microsoft-EmptyAppServicePlans.json b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Recommendations/queries/Recommendations-Microsoft-EmptyAppServicePlans.json new file mode 100644 index 000000000..3306b29fb --- /dev/null +++ b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Recommendations/queries/Recommendations-Microsoft-EmptyAppServicePlans.json @@ -0,0 +1,10 @@ +{ + "dataset": "Recommendations", + "provider": "Microsoft", + "query": "resources | where type =~ 'microsoft.web/serverfarms' | where toint(properties.numberOfSites) == 0 | where sku.tier !~ 'Free' | extend Location=location, SKUName=sku.name, SKUTier=sku.tier, SKUCapacity=tostring(sku.capacity) | project id, name, resourceGroup, subscriptionId, Location, SKUName, SKUTier, SKUCapacity, type | project x_RecommendationId=strcat(tolower(id),'-emptyAppServicePlan'), x_ResourceGroupName=tolower(resourceGroup), SubAccountId=subscriptionId, x_RecommendationCategory='Cost', x_RecommendationDescription='App Service plan has no apps hosted on it', ResourceId = tolower(id), ResourceName=tolower(name), x_RecommendationDetails = tostring(bag_pack('SKUName', SKUName, 'SKUTier', SKUTier, 'SKUCapacity', SKUCapacity, 'Location', Location, 'x_RecommendationProvider', 'FinOps hubs', 'x_RecommendationSolution', 'Delete the empty App Service plan or deploy apps to it', 'x_RecommendationTypeId', 'd7b1c874-5e1a-4e3b-9f0a-1c2d3e4f5a6b', 'x_ResourceType', type)), x_RecommendationDate = now() | join kind=leftouter ( resourcecontainers | where type == 'microsoft.resources/subscriptions' | project SubAccountName=name, SubAccountId=subscriptionId ) on SubAccountId | project-away SubAccountId1", + "queryEngine": "ResourceGraph", + "scope": "Tenant", + "source": "FinOps hubs", + "type": "Microsoft-EmptyAppServicePlans", + "version": "1.0" +} diff --git a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Recommendations/queries/Recommendations-Microsoft-EmptyNSGs.json b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Recommendations/queries/Recommendations-Microsoft-EmptyNSGs.json new file mode 100644 index 000000000..f5a9571e0 --- /dev/null +++ b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Recommendations/queries/Recommendations-Microsoft-EmptyNSGs.json @@ -0,0 +1,10 @@ +{ + "dataset": "Recommendations", + "provider": "Microsoft", + "query": "resources | where type =~ 'microsoft.network/networksecuritygroups' | where isnull(properties.networkInterfaces) and isnull(properties.subnets) | extend Location=location | project id, name, resourceGroup, subscriptionId, Location, type | project x_RecommendationId=strcat(tolower(id),'-emptyNSG'), x_ResourceGroupName=tolower(resourceGroup), SubAccountId=subscriptionId, x_RecommendationCategory='Cost', x_RecommendationDescription='Network security group is not associated with any network interface or subnet', ResourceId = tolower(id), ResourceName=tolower(name), x_RecommendationDetails = tostring(bag_pack('Location', Location, 'x_RecommendationProvider', 'FinOps hubs', 'x_RecommendationSolution', 'Delete the orphaned network security group or associate it with a subnet or NIC', 'x_RecommendationTypeId', 'b9c0d1e2-f3a4-5b6c-7d8e-9f0a1b2c3d4e', 'x_ResourceType', type)), x_RecommendationDate = now() | join kind=leftouter ( resourcecontainers | where type == 'microsoft.resources/subscriptions' | project SubAccountName=name, SubAccountId=subscriptionId ) on SubAccountId | project-away SubAccountId1", + "queryEngine": "ResourceGraph", + "scope": "Tenant", + "source": "FinOps hubs", + "type": "Microsoft-EmptyNSGs", + "version": "1.0" +} diff --git a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Recommendations/queries/Recommendations-Microsoft-EmptySQLElasticPools.json b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Recommendations/queries/Recommendations-Microsoft-EmptySQLElasticPools.json new file mode 100644 index 000000000..07fc4725c --- /dev/null +++ b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Recommendations/queries/Recommendations-Microsoft-EmptySQLElasticPools.json @@ -0,0 +1,10 @@ +{ + "dataset": "Recommendations", + "provider": "Microsoft", + "query": "resources | where type == 'microsoft.sql/servers/elasticpools'| extend elasticPoolId = tolower(tostring(id)), elasticPoolName = name, elasticPoolRG = resourceGroup,skuName=tostring(sku.name),skuTier=tostring(sku.tier),skuCapacity=tostring(sku.capacity), Location=location, type| join kind=leftouter ( resources | where type == 'microsoft.sql/servers/databases'| extend elasticPoolId = tolower(tostring(properties.elasticPoolId)) ) on elasticPoolId| summarize databaseCount = countif(isnotempty(elasticPoolId1)) by elasticPoolId, elasticPoolName,serverResourceGroup=resourceGroup,name,skuName,skuTier,skuCapacity,elasticPoolRG,Location, type, subscriptionId| where databaseCount == 0 | project elasticPoolId, elasticPoolName, databaseCount, elasticPoolRG ,skuName,skuTier ,skuCapacity, Location, type, subscriptionId| project x_RecommendationId=strcat(tolower(elasticPoolId),'-idle'), x_ResourceGroupName=tolower(elasticPoolRG), SubAccountId=subscriptionId, x_RecommendationCategory='Cost', x_RecommendationDescription='SQL Database elastic pool has no associated databases', ResourceId = tolower(elasticPoolId), ResourceName=tolower(elasticPoolName), x_RecommendationDetails= tostring(bag_pack('skuName', skuName, 'skuTier', skuTier, 'skuCapacity', skuCapacity, 'Location', Location, 'x_RecommendationProvider', 'FinOps hubs', 'x_RecommendationSolution', 'Review and remove this resource if not needed', 'x_RecommendationTypeId', '50987aae-a46d-49ae-bd41-a670a4dd18bd', 'x_ResourceType', type)), x_RecommendationDate = now() | join kind=leftouter ( resourcecontainers | where type == 'microsoft.resources/subscriptions' | project SubAccountName=name, SubAccountId=subscriptionId ) on SubAccountId | project-away SubAccountId1", + "queryEngine": "ResourceGraph", + "scope": "Tenant", + "source": "FinOps hubs", + "type": "Microsoft-EmptySQLElasticPools", + "version": "1.0" +} diff --git a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Recommendations/queries/Recommendations-Microsoft-IdleVNetGateways.json b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Recommendations/queries/Recommendations-Microsoft-IdleVNetGateways.json new file mode 100644 index 000000000..083551785 --- /dev/null +++ b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Recommendations/queries/Recommendations-Microsoft-IdleVNetGateways.json @@ -0,0 +1,10 @@ +{ + "dataset": "Recommendations", + "provider": "Microsoft", + "query": "resources | where type =~ 'microsoft.network/virtualnetworkgateways' | extend gatewayType = tostring(properties.gatewayType) | where gatewayType in ('Vpn', 'ExpressRoute') | mv-expand ipConfig = properties.ipConfigurations | extend pipId = tostring(ipConfig.properties.publicIPAddress.id) | extend gwId = tolower(id) | join kind=leftouter ( resources | where type =~ 'microsoft.network/connections' | mv-expand gwRef = pack_array(properties.virtualNetworkGateway1.id, properties.virtualNetworkGateway2.id) | extend gwId = tolower(tostring(gwRef)) | summarize connectionCount = count() by gwId ) on gwId | where isnull(connectionCount) or connectionCount == 0 | extend Location=location, SKUName=tostring(properties.sku.name), SKUTier=tostring(properties.sku.tier) | project id, name, resourceGroup, subscriptionId, Location, SKUName, SKUTier, gatewayType, type | project x_RecommendationId=strcat(tolower(id),'-idleVNetGateway'), x_ResourceGroupName=tolower(resourceGroup), SubAccountId=subscriptionId, x_RecommendationCategory='Cost', x_RecommendationDescription='Virtual network gateway has no connections', ResourceId = tolower(id), ResourceName=tolower(name), x_RecommendationDetails = tostring(bag_pack('GatewayType', gatewayType, 'SKUName', SKUName, 'SKUTier', SKUTier, 'Location', Location, 'x_RecommendationProvider', 'FinOps hubs', 'x_RecommendationSolution', 'Delete the idle virtual network gateway or add connections', 'x_RecommendationTypeId', 'a3b4c5d6-7e8f-9a0b-1c2d-3e4f5a6b7c8d', 'x_ResourceType', type)), x_RecommendationDate = now() | join kind=leftouter ( resourcecontainers | where type == 'microsoft.resources/subscriptions' | project SubAccountName=name, SubAccountId=subscriptionId ) on SubAccountId | project-away SubAccountId1", + "queryEngine": "ResourceGraph", + "scope": "Tenant", + "source": "FinOps hubs", + "type": "Microsoft-IdleVNetGateways", + "version": "1.0" +} diff --git a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Recommendations/queries/Recommendations-Microsoft-LegacyStorageAccounts.json b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Recommendations/queries/Recommendations-Microsoft-LegacyStorageAccounts.json new file mode 100644 index 000000000..b48ed28fe --- /dev/null +++ b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Recommendations/queries/Recommendations-Microsoft-LegacyStorageAccounts.json @@ -0,0 +1,10 @@ +{ + "dataset": "Recommendations", + "provider": "Microsoft", + "query": "resources | where type =~ 'microsoft.storage/storageaccounts' | where kind in~ ('Storage', 'BlobStorage') and parse_json(sku).name in~ ('Standard_LRS', 'Standard_GRS', 'Standard_ZRS', 'Standard_RAGRS', 'Standard_RAGZRS') | extend Location=location, AccessTier=tostring(properties.accessTier), Kind=tostring(kind) | project id, name, resourceGroup, subscriptionId, Location, Kind, AccessTier, type | project x_RecommendationId=strcat(tolower(id),'-legacyStorageAccount'), x_ResourceGroupName=tolower(resourceGroup), SubAccountId=subscriptionId, x_RecommendationCategory='Cost', x_RecommendationDescription='Storage account is using a legacy kind that is being retired', ResourceId = tolower(id), ResourceName=tolower(name), x_RecommendationDetails = tostring(bag_pack('Kind', Kind, 'AccessTier', AccessTier, 'Location', Location, 'x_RecommendationProvider', 'FinOps hubs', 'x_RecommendationSolution', 'Upgrade to General Purpose v2 (StorageV2) storage account', 'x_RecommendationTypeId', 'c4d5e6f7-a8b9-0c1d-2e3f-4a5b6c7d8e9f', 'x_ResourceType', type)), x_RecommendationDate = now() | join kind=leftouter ( resourcecontainers | where type == 'microsoft.resources/subscriptions' | project SubAccountName=name, SubAccountId=subscriptionId ) on SubAccountId | project-away SubAccountId1", + "queryEngine": "ResourceGraph", + "scope": "Tenant", + "source": "FinOps hubs", + "type": "Microsoft-LegacyStorageAccounts", + "version": "1.0" +} diff --git a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Recommendations/queries/Recommendations-Microsoft-NonSpotAKSClusters.json b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Recommendations/queries/Recommendations-Microsoft-NonSpotAKSClusters.json new file mode 100644 index 000000000..2f2137a94 --- /dev/null +++ b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Recommendations/queries/Recommendations-Microsoft-NonSpotAKSClusters.json @@ -0,0 +1,11 @@ +{ + "dataset": "Recommendations", + "provider": "Microsoft", + "query": "resources | where type == 'microsoft.containerservice/managedclusters' | mvexpand AgentPoolProfiles = properties.agentPoolProfiles| project id, type, ProfileName = tostring(AgentPoolProfiles.name), Sku = tostring(sku.name), Tier = tostring(sku.tier), mode = AgentPoolProfiles.mode, AutoScaleEnabled = AgentPoolProfiles.enableAutoScaling, SpotVM = AgentPoolProfiles.scaleSetPriority, VMSize = tostring(AgentPoolProfiles.vmSize), NodeCount = tostring(AgentPoolProfiles.['count']), minCount = tostring(AgentPoolProfiles.minCount), maxCount = tostring(AgentPoolProfiles.maxCount), Location=location, resourceGroup, subscriptionId, AKSname = name| where AutoScaleEnabled == 'true' and isnull(SpotVM)| project x_RecommendationId=strcat(tolower(id),'-notSpot'), x_ResourceGroupName=tolower(resourceGroup), SubAccountId=subscriptionId, x_RecommendationCategory='Cost', x_RecommendationDescription='The AKS cluster agent pool scale set is not utilizing Spot VMs', ResourceId = tolower(id), ResourceName=tolower(AKSname), x_RecommendationDetails= tostring(bag_pack('AutoScaleEnabled', AutoScaleEnabled, 'SpotVM', SpotVM, 'VMSize', VMSize, 'Location', Location, 'NodeCount', NodeCount, 'minCount', minCount, 'maxCount', maxCount, 'x_RecommendationProvider', 'FinOps hubs', 'x_RecommendationSolution', 'Consider enabling Spot VMs for this AKS cluster to optimize costs, as Spot VMs offer significantly lower pricing compared to regular VMs', 'x_RecommendationTypeId', 'c26abcc2-d5e6-4654-be4a-7d338e5c1e5f', 'x_ResourceType', type)), x_RecommendationDate = now() | join kind=leftouter ( resourcecontainers | where type == 'microsoft.resources/subscriptions' | project SubAccountName=name, SubAccountId=subscriptionId ) on SubAccountId | project-away SubAccountId1", + "queryEngine": "ResourceGraph", + "scope": "Tenant", + "source": "FinOps hubs", + "group": "spot", + "type": "Microsoft-NonSpotAKSClusters", + "version": "1.0" +} diff --git a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Recommendations/queries/Recommendations-Microsoft-OrphanedNATGateways.json b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Recommendations/queries/Recommendations-Microsoft-OrphanedNATGateways.json new file mode 100644 index 000000000..535aadcc4 --- /dev/null +++ b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Recommendations/queries/Recommendations-Microsoft-OrphanedNATGateways.json @@ -0,0 +1,10 @@ +{ + "dataset": "Recommendations", + "provider": "Microsoft", + "query": "resources | where type =~ 'microsoft.network/natgateways' | where isnull(properties.subnets) or array_length(properties.subnets) == 0 | extend Location=location, SKUName=tostring(sku.name) | project id, name, resourceGroup, subscriptionId, Location, SKUName, type | project x_RecommendationId=strcat(tolower(id),'-orphanedNATGateway'), x_ResourceGroupName=tolower(resourceGroup), SubAccountId=subscriptionId, x_RecommendationCategory='Cost', x_RecommendationDescription='NAT gateway is not associated with any subnet', ResourceId = tolower(id), ResourceName=tolower(name), x_RecommendationDetails = tostring(bag_pack('SKUName', SKUName, 'Location', Location, 'x_RecommendationProvider', 'FinOps hubs', 'x_RecommendationSolution', 'Delete the orphaned NAT gateway or associate it with a subnet', 'x_RecommendationTypeId', 'b4c5d6e7-8f9a-0b1c-2d3e-4f5a6b7c8d9e', 'x_ResourceType', type)), x_RecommendationDate = now() | join kind=leftouter ( resourcecontainers | where type == 'microsoft.resources/subscriptions' | project SubAccountName=name, SubAccountId=subscriptionId ) on SubAccountId | project-away SubAccountId1", + "queryEngine": "ResourceGraph", + "scope": "Tenant", + "source": "FinOps hubs", + "type": "Microsoft-OrphanedNATGateways", + "version": "1.0" +} diff --git a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Recommendations/queries/Recommendations-Microsoft-PremiumSnapshots.json b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Recommendations/queries/Recommendations-Microsoft-PremiumSnapshots.json new file mode 100644 index 000000000..d6f264914 --- /dev/null +++ b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Recommendations/queries/Recommendations-Microsoft-PremiumSnapshots.json @@ -0,0 +1,10 @@ +{ + "dataset": "Recommendations", + "provider": "Microsoft", + "query": "resources | where type =~ 'microsoft.compute/snapshots' | where sku.name =~ 'Premium_LRS' | extend DiskSizeGB=tostring(properties.diskSizeGB), TimeCreated=tostring(properties.timeCreated), Location=location, SKUName=tostring(sku.name) | project id, name, resourceGroup, subscriptionId, Location, DiskSizeGB, TimeCreated, SKUName, type | project x_RecommendationId=strcat(tolower(id),'-premiumSnapshot'), x_ResourceGroupName=tolower(resourceGroup), SubAccountId=subscriptionId, x_RecommendationCategory='Cost', x_RecommendationDescription='Snapshot is using Premium SSD storage instead of Standard', ResourceId = tolower(id), ResourceName=tolower(name), x_RecommendationDetails = tostring(bag_pack('DiskSizeGB', DiskSizeGB, 'SKUName', SKUName, 'TimeCreated', TimeCreated, 'Location', Location, 'x_RecommendationProvider', 'FinOps hubs', 'x_RecommendationSolution', 'Change snapshot storage type from Premium SSD to Standard HDD', 'x_RecommendationTypeId', 'e7f8a9b0-1c2d-3e4f-5a6b-7c8d9e0f1a2b', 'x_ResourceType', type)), x_RecommendationDate = now() | join kind=leftouter ( resourcecontainers | where type == 'microsoft.resources/subscriptions' | project SubAccountName=name, SubAccountId=subscriptionId ) on SubAccountId | project-away SubAccountId1", + "queryEngine": "ResourceGraph", + "scope": "Tenant", + "source": "FinOps hubs", + "type": "Microsoft-PremiumSnapshots", + "version": "1.0" +} diff --git a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Recommendations/queries/Recommendations-Microsoft-SQLVMsWithoutAHB.json b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Recommendations/queries/Recommendations-Microsoft-SQLVMsWithoutAHB.json new file mode 100644 index 000000000..1e4691147 --- /dev/null +++ b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Recommendations/queries/Recommendations-Microsoft-SQLVMsWithoutAHB.json @@ -0,0 +1,11 @@ +{ + "dataset": "Recommendations", + "provider": "Microsoft", + "query": "resourcecontainers | where type =~ 'Microsoft.Resources/subscriptions' | where tostring (properties.subscriptionPolicies.quotaId) !has 'MSDNDevTest_2014-09-01' | extend SubscriptionName=name | join ( resources | where type =~ 'Microsoft.SqlVirtualMachine/SqlVirtualMachines' and tostring(properties.['sqlServerLicenseType']) != 'AHUB' | extend SQLID=id, VMName = name, resourceGroup, Location = location, LicenseType = tostring(properties.['sqlServerLicenseType']), OSType=tostring(properties.storageProfile.imageReference.offer), SQLAgentType = tostring(properties.['sqlManagement']), SQLVersion = tostring(properties.['sqlImageOffer']), SQLSKU=tostring(properties.['sqlImageSku'])) on subscriptionId | join ( resources | where type =~ 'Microsoft.Compute/virtualmachines' | extend ActualCores = toint(extract('.[A-Z]([0-9]+)', 1, tostring(properties.hardwareProfile.vmSize))) | project VMName = tolower(name), VMSize = tostring(properties.hardwareProfile.vmSize),ActualCores, subscriptionId, vmType=type, vmResourceGroup=resourceGroup) on VMName| order by id asc | where SQLSKU != 'Developer' and SQLSKU != 'Express'| extend CheckAHBSQLVM= case( type == 'Microsoft.SqlVirtualMachine/SqlVirtualMachines', iif((properties.['sqlServerLicenseType']) != 'AHUB', 'AHB-disabled', 'AHB-enabled'), 'Not Windows')| project SQLID,VMName,resourceGroup, Location, VMSize, SQLVersion, SQLSKU, SQLAgentType, LicenseType, SubscriptionName,type,CheckAHBSQLVM, subscriptionId,ActualCores, vmType, vmResourceGroup| project x_RecommendationId=strcat(tolower(SQLID),'-CheckAHBSQLVM'), x_ResourceGroupName=tolower(vmResourceGroup), SubAccountId=subscriptionId, x_RecommendationCategory='Cost', x_RecommendationDescription='SQL virtual machine is not leveraging Azure Hybrid Benefit', ResourceId = tolower(SQLID), ResourceName=tolower(VMName), x_RecommendationDetails= tostring(bag_pack('VMSize', VMSize, 'CheckAHBSQLVM', CheckAHBSQLVM, 'ActualCores', ActualCores, 'SQLVersion', SQLVersion, 'SQLSKU', SQLSKU, 'SQLAgentType', SQLAgentType, 'LicenseType', LicenseType, 'Location', Location, 'x_RecommendationProvider', 'FinOps hubs', 'x_RecommendationSolution', 'Review the SQL licensing option', 'x_RecommendationTypeId', '01decd62-f91b-4434-abe5-9a09e13e018f', 'x_ResourceType', vmType)), x_RecommendationDate = now() | join kind=leftouter ( resourcecontainers | where type == 'microsoft.resources/subscriptions' | project SubAccountName=name, SubAccountId=subscriptionId ) on SubAccountId | project-away SubAccountId1", + "queryEngine": "ResourceGraph", + "scope": "Tenant", + "source": "FinOps hubs", + "group": "ahb", + "type": "Microsoft-SQLVMsWithoutAHB", + "version": "1.0" +} diff --git a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Recommendations/queries/Recommendations-Microsoft-StoppedVMs.json b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Recommendations/queries/Recommendations-Microsoft-StoppedVMs.json new file mode 100644 index 000000000..4834d0271 --- /dev/null +++ b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Recommendations/queries/Recommendations-Microsoft-StoppedVMs.json @@ -0,0 +1,10 @@ +{ + "dataset": "Recommendations", + "provider": "Microsoft", + "query": "resources | where type =~ 'microsoft.compute/virtualmachines' and tolower(tostring(properties.extended.instanceView.powerState.code)) =~ 'powerstate/stopped' | extend PowerState=tostring(properties.extended.instanceView.powerState.displayStatus) | extend Location=location, type| project id, PowerState, Location, resourceGroup, subscriptionId, VMName=name, type| project x_RecommendationId=strcat(tolower(id),'-notDeallocated'), x_ResourceGroupName=tolower(resourceGroup), SubAccountId=subscriptionId, x_RecommendationCategory='Cost', x_RecommendationDescription='Virtual machine is powered off but not deallocated', ResourceId = tolower(id), ResourceName=tolower(VMName), x_RecommendationDetails= tostring(bag_pack('PowerState', PowerState, 'Location', Location, 'x_RecommendationProvider', 'FinOps hubs', 'x_RecommendationSolution', 'Deallocate the virtual machine to ensure it does not incur in compute costs', 'x_RecommendationTypeId', 'ab2ff882-e927-4093-9d11-631be0219975', 'x_ResourceType', type)), x_RecommendationDate = now() | join kind=leftouter ( resourcecontainers | where type == 'microsoft.resources/subscriptions' | project SubAccountName=name, SubAccountId=subscriptionId ) on SubAccountId | project-away SubAccountId1", + "queryEngine": "ResourceGraph", + "scope": "Tenant", + "source": "FinOps hubs", + "type": "Microsoft-StoppedVMs", + "version": "1.0" +} diff --git a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Recommendations/queries/Recommendations-Microsoft-UnassociatedDDoSPlans.json b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Recommendations/queries/Recommendations-Microsoft-UnassociatedDDoSPlans.json new file mode 100644 index 000000000..5e1843b3b --- /dev/null +++ b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Recommendations/queries/Recommendations-Microsoft-UnassociatedDDoSPlans.json @@ -0,0 +1,10 @@ +{ + "dataset": "Recommendations", + "provider": "Microsoft", + "query": "resources | where type =~ 'microsoft.network/ddosprotectionplans' | where isnull(properties.virtualNetworks) or array_length(properties.virtualNetworks) == 0 | extend Location=location | project id, name, resourceGroup, subscriptionId, Location, type | project x_RecommendationId=strcat(tolower(id),'-unassociatedDDoSPlan'), x_ResourceGroupName=tolower(resourceGroup), SubAccountId=subscriptionId, x_RecommendationCategory='Cost', x_RecommendationDescription='DDoS protection plan is not associated with any virtual network', ResourceId = tolower(id), ResourceName=tolower(name), x_RecommendationDetails = tostring(bag_pack('Location', Location, 'x_RecommendationProvider', 'FinOps hubs', 'x_RecommendationSolution', 'Delete the DDoS protection plan or associate it with virtual networks', 'x_RecommendationTypeId', 'c5d6e7f8-9a0b-1c2d-3e4f-5a6b7c8d9e0f', 'x_ResourceType', type)), x_RecommendationDate = now() | join kind=leftouter ( resourcecontainers | where type == 'microsoft.resources/subscriptions' | project SubAccountName=name, SubAccountId=subscriptionId ) on SubAccountId | project-away SubAccountId1", + "queryEngine": "ResourceGraph", + "scope": "Tenant", + "source": "FinOps hubs", + "type": "Microsoft-UnassociatedDDoSPlans", + "version": "1.0" +} diff --git a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Recommendations/queries/Recommendations-Microsoft-UnattachedDisks.json b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Recommendations/queries/Recommendations-Microsoft-UnattachedDisks.json new file mode 100644 index 000000000..939aec12c --- /dev/null +++ b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Recommendations/queries/Recommendations-Microsoft-UnattachedDisks.json @@ -0,0 +1,10 @@ +{ + "dataset": "Recommendations", + "provider": "Microsoft", + "query": "resources | where type =~ 'microsoft.compute/disks' and isempty(managedBy) | extend diskState = tostring(properties.diskState) | where diskState != 'ActiveSAS' and tags !contains 'ASR-ReplicaDisk' and tags !contains 'asrseeddisk' | extend DiskId=id, DiskIDfull=id, DiskName=name, SKUName=sku.name, SKUTier=sku.tier, DiskSizeGB=tostring(properties.diskSizeGB), Location=location, TimeCreated=tostring(properties.timeCreated), SubId=subscriptionId | order by DiskId asc | project DiskId, DiskIDfull, DiskName, DiskSizeGB, SKUName, SKUTier, resourceGroup, Location, TimeCreated, subscriptionId, type| project x_RecommendationId=strcat(tolower(DiskId),'-unattached'), x_ResourceGroupName=tolower(resourceGroup), SubAccountId=subscriptionId, x_RecommendationCategory='Cost', x_RecommendationDescription='Unattached (orphaned) disk is incurring in storage costs', ResourceId = tolower(DiskId), ResourceName=tolower(DiskName), x_RecommendationDetails= tostring(bag_pack('DiskSizeGB', DiskSizeGB, 'SKUName', SKUName, 'SKUTier', SKUTier, 'Location', Location, 'TimeCreated', TimeCreated, 'x_RecommendationProvider', 'FinOps hubs', 'x_RecommendationSolution', 'Remove or downgrade the unattached disk', 'x_RecommendationTypeId', 'e0c02939-ce02-4f9d-881f-8067ae7ec90f', 'x_ResourceType', type)), x_RecommendationDate = now() | join kind=leftouter ( resourcecontainers | where type == 'microsoft.resources/subscriptions' | project SubAccountName=name, SubAccountId=subscriptionId ) on SubAccountId | project-away SubAccountId1", + "queryEngine": "ResourceGraph", + "scope": "Tenant", + "source": "FinOps hubs", + "type": "Microsoft-UnattachedDisks", + "version": "1.0" +} diff --git a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Recommendations/queries/Recommendations-Microsoft-UnattachedNICs.json b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Recommendations/queries/Recommendations-Microsoft-UnattachedNICs.json new file mode 100644 index 000000000..d838d8cc6 --- /dev/null +++ b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Recommendations/queries/Recommendations-Microsoft-UnattachedNICs.json @@ -0,0 +1,10 @@ +{ + "dataset": "Recommendations", + "provider": "Microsoft", + "query": "resources | where type =~ 'microsoft.network/networkinterfaces' | where isnull(properties.virtualMachine) and isnull(properties.privateEndpoint) | extend Location=location, PrivateIP=tostring(properties.ipConfigurations[0].properties.privateIPAddress) | project id, name, resourceGroup, subscriptionId, Location, PrivateIP, type | project x_RecommendationId=strcat(tolower(id),'-unattachedNIC'), x_ResourceGroupName=tolower(resourceGroup), SubAccountId=subscriptionId, x_RecommendationCategory='Cost', x_RecommendationDescription='Network interface is not attached to any virtual machine or private endpoint', ResourceId = tolower(id), ResourceName=tolower(name), x_RecommendationDetails = tostring(bag_pack('PrivateIP', PrivateIP, 'Location', Location, 'x_RecommendationProvider', 'FinOps hubs', 'x_RecommendationSolution', 'Delete the orphaned network interface', 'x_RecommendationTypeId', 'a8b9c0d1-e2f3-4a5b-6c7d-8e9f0a1b2c3d', 'x_ResourceType', type)), x_RecommendationDate = now() | join kind=leftouter ( resourcecontainers | where type == 'microsoft.resources/subscriptions' | project SubAccountName=name, SubAccountId=subscriptionId ) on SubAccountId | project-away SubAccountId1", + "queryEngine": "ResourceGraph", + "scope": "Tenant", + "source": "FinOps hubs", + "type": "Microsoft-UnattachedNICs", + "version": "1.0" +} diff --git a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Recommendations/queries/Recommendations-Microsoft-UnattachedPublicIPs.json b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Recommendations/queries/Recommendations-Microsoft-UnattachedPublicIPs.json new file mode 100644 index 000000000..67cc597b4 --- /dev/null +++ b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Recommendations/queries/Recommendations-Microsoft-UnattachedPublicIPs.json @@ -0,0 +1,10 @@ +{ + "dataset": "Recommendations", + "provider": "Microsoft", + "query": "resources | where type =~ 'Microsoft.Network/publicIPAddresses' and isempty(properties.ipConfiguration) and isempty(properties.natGateway) | extend PublicIpId=id, IPName=name, AllocationMethod=tostring(properties.publicIPAllocationMethod), SKUName=sku.name, Location=location | project PublicIpId, IPName, SKUName, resourceGroup, Location, AllocationMethod, subscriptionId, type, name | union ( resources | where type =~ 'microsoft.network/networkinterfaces' and isempty(properties.virtualMachine) and isnull(properties.privateEndpoint) and isnotempty(properties.ipConfigurations) | extend IPconfig = properties.ipConfigurations | mv-expand IPconfig | extend PublicIpId= tostring(IPconfig.properties.publicIPAddress.id) | project PublicIpId, name | join ( resources | where type =~ 'Microsoft.Network/publicIPAddresses'| extend PublicIpId=id, IPName=name, AllocationMethod=tostring(properties.publicIPAllocationMethod), SKUName=sku.name, resourceGroup, Location=location, name, id ) on PublicIpId | extend PublicIpId,IPName, SKUName, resourceGroup, Location, AllocationMethod,name, subscriptionId )| project x_RecommendationId=strcat(tolower(PublicIpId),'-idle'), x_ResourceGroupName=tolower(resourceGroup), SubAccountId=subscriptionId, x_RecommendationCategory='Cost', x_RecommendationDescription='Unattached (orphaned) public IP', ResourceId = tolower(PublicIpId), ResourceName=tolower(name), x_RecommendationDetails= tostring(bag_pack('SKUName', SKUName, 'AllocationMethod', AllocationMethod, 'Location', Location, 'x_RecommendationProvider', 'FinOps hubs', 'x_RecommendationSolution', 'Review and remove this resource if not needed', 'x_RecommendationTypeId', '3ecbf770-9404-4504-a450-cc198e8b2a7d', 'x_ResourceType', type)), x_RecommendationDate = now() | join kind=leftouter ( resourcecontainers | where type == 'microsoft.resources/subscriptions' | project SubAccountName=name, SubAccountId=subscriptionId ) on SubAccountId | project-away SubAccountId1", + "queryEngine": "ResourceGraph", + "scope": "Tenant", + "source": "FinOps hubs", + "type": "Microsoft-UnattachedPublicIPs", + "version": "1.0" +} diff --git a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Recommendations/queries/Recommendations-Microsoft-UnmanagedDisks.json b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Recommendations/queries/Recommendations-Microsoft-UnmanagedDisks.json new file mode 100644 index 000000000..36839eddf --- /dev/null +++ b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Recommendations/queries/Recommendations-Microsoft-UnmanagedDisks.json @@ -0,0 +1,10 @@ +{ + "dataset": "Recommendations", + "provider": "Microsoft", + "query": "resources | where type =~ 'microsoft.compute/virtualmachines' | where isnull(properties.storageProfile.osDisk.managedDisk) | extend Location=location, VMSize=tostring(properties.hardwareProfile.vmSize) | project id, name, resourceGroup, subscriptionId, Location, VMSize, type | project x_RecommendationId=strcat(tolower(id),'-unmanagedDisks'), x_ResourceGroupName=tolower(resourceGroup), SubAccountId=subscriptionId, x_RecommendationCategory='Cost', x_RecommendationDescription='Virtual machine is using unmanaged disks that are being retired', ResourceId = tolower(id), ResourceName=tolower(name), x_RecommendationDetails = tostring(bag_pack('VMSize', VMSize, 'Location', Location, 'x_RecommendationProvider', 'FinOps hubs', 'x_RecommendationSolution', 'Migrate to managed disks', 'x_RecommendationTypeId', 'd5e6f7a8-b9c0-1d2e-3f4a-5b6c7d8e9f0a', 'x_ResourceType', type)), x_RecommendationDate = now() | join kind=leftouter ( resourcecontainers | where type == 'microsoft.resources/subscriptions' | project SubAccountName=name, SubAccountId=subscriptionId ) on SubAccountId | project-away SubAccountId1", + "queryEngine": "ResourceGraph", + "scope": "Tenant", + "source": "FinOps hubs", + "type": "Microsoft-UnmanagedDisks", + "version": "1.0" +} diff --git a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Recommendations/queries/Recommendations-Microsoft-UnprovisionedExpressRouteCircuits.json b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Recommendations/queries/Recommendations-Microsoft-UnprovisionedExpressRouteCircuits.json new file mode 100644 index 000000000..ed1a72602 --- /dev/null +++ b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Recommendations/queries/Recommendations-Microsoft-UnprovisionedExpressRouteCircuits.json @@ -0,0 +1,10 @@ +{ + "dataset": "Recommendations", + "provider": "Microsoft", + "query": "resources | where type =~ 'microsoft.network/expressroutecircuits' | where tostring(properties.serviceProviderProvisioningState) != 'Provisioned' | extend CircuitState=tostring(properties.circuitProvisioningState), ProviderState=tostring(properties.serviceProviderProvisioningState), ServiceProvider=tostring(properties.serviceProviderProperties.serviceProviderName), BandwidthMbps=tostring(properties.serviceProviderProperties.bandwidthInMbps), SKUName=tostring(sku.name), SKUTier=tostring(sku.tier), SKUFamily=tostring(sku.family), Location=location | project id, name, resourceGroup, subscriptionId, Location, CircuitState, ProviderState, ServiceProvider, BandwidthMbps, SKUName, SKUTier, SKUFamily, type | project x_RecommendationId=strcat(tolower(id),'-unprovisionedExpressRoute'), x_ResourceGroupName=tolower(resourceGroup), SubAccountId=subscriptionId, x_RecommendationCategory='Cost', x_RecommendationDescription='ExpressRoute circuit is not in provisioned state', ResourceId = tolower(id), ResourceName=tolower(name), x_RecommendationDetails = tostring(bag_pack('CircuitState', CircuitState, 'ProviderState', ProviderState, 'ServiceProvider', ServiceProvider, 'BandwidthMbps', BandwidthMbps, 'SKUName', SKUName, 'SKUTier', SKUTier, 'SKUFamily', SKUFamily, 'Location', Location, 'x_RecommendationProvider', 'FinOps hubs', 'x_RecommendationSolution', 'Complete provisioning or delete the ExpressRoute circuit', 'x_RecommendationTypeId', 'd6e7f8a9-0b1c-2d3e-4f5a-6b7c8d9e0f1a', 'x_ResourceType', type)), x_RecommendationDate = now() | join kind=leftouter ( resourcecontainers | where type == 'microsoft.resources/subscriptions' | project SubAccountName=name, SubAccountId=subscriptionId ) on SubAccountId | project-away SubAccountId1", + "queryEngine": "ResourceGraph", + "scope": "Tenant", + "source": "FinOps hubs", + "type": "Microsoft-UnprovisionedExpressRouteCircuits", + "version": "1.0" +} diff --git a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Recommendations/queries/Recommendations-Microsoft-VMsWithoutAHB.json b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Recommendations/queries/Recommendations-Microsoft-VMsWithoutAHB.json new file mode 100644 index 000000000..603771ebb --- /dev/null +++ b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Recommendations/queries/Recommendations-Microsoft-VMsWithoutAHB.json @@ -0,0 +1,11 @@ +{ + "dataset": "Recommendations", + "provider": "Microsoft", + "query": "resourcecontainers | where type =~ 'Microsoft.Resources/subscriptions' | where tostring (properties.subscriptionPolicies.quotaId) !has 'MSDNDevTest_2014-09-01' | extend SubscriptionName=name | join ( resources | where type =~ 'microsoft.compute/virtualmachines' or type =~ 'microsoft.compute/virtualMachineScaleSets' | where (tostring(properties.virtualMachineProfile.storageProfile.osDisk.osType) == 'Windows' and tostring(properties.virtualMachineProfile.storageProfile.imageReference.publisher) !in~ ('microsoftwindowsdesktop','microsoftvisualstudio') and tostring(properties.virtualMachineProfile.licenseType) !startswith 'Windows') or (tostring(properties.storageProfile.osDisk.osType) == 'Windows' and tostring(properties.storageProfile.imageReference.publisher) !in~ ('microsoftwindowsdesktop','microsoftvisualstudio') and tostring(properties.licenseType) !startswith 'Windows') | extend WindowsId=id, VMSku=tostring(properties.hardwareProfile.vmSize), vmResourceGroup=resourceGroup, vmType=type, Location=location,LicenseType = tostring(properties.['licenseType']) | extend ActualCores = toint(extract('.[A-Z]([0-9]+)', 1, tostring(properties.hardwareProfile.vmSize))) | extend CheckAHBWindows = case( type == 'microsoft.compute/virtualmachines' or type =~ 'microsoft.compute/virtualMachineScaleSets', iif((properties.['licenseType']) !has 'Windows' and (properties.virtualMachineProfile.['licenseType']) !has 'Windows' , 'AHB-disabled', 'AHB-enabled'), 'Not Windows' )) on subscriptionId | project x_RecommendationId=strcat(tolower(WindowsId),'-CheckAHBWindows'), x_ResourceGroupName=tolower(vmResourceGroup), SubAccountId=subscriptionId, x_RecommendationCategory='Cost', x_RecommendationDescription='Windows virtual machine is not leveraging Azure Hybrid Benefit', ResourceId = tolower(WindowsId), ResourceName=tolower(split(WindowsId,'/')[-1]), x_RecommendationDetails= tostring(bag_pack('VMSku', VMSku, 'CheckAHBWindows', CheckAHBWindows, 'ActualCores', ActualCores, 'Location', Location, 'x_RecommendationProvider', 'FinOps hubs', 'x_RecommendationSolution', 'Review the virtual machine licensing option', 'x_RecommendationTypeId', 'f326c065-b9f7-4a0e-a0f1-5a1c060bc25d', 'x_ResourceType', vmType)), x_RecommendationDate = now() | join kind=leftouter ( resourcecontainers | where type == 'microsoft.resources/subscriptions' | project SubAccountName=name, SubAccountId=subscriptionId ) on SubAccountId | project-away SubAccountId1", + "queryEngine": "ResourceGraph", + "scope": "Tenant", + "source": "FinOps hubs", + "group": "ahb", + "type": "Microsoft-VMsWithoutAHB", + "version": "1.0" +} diff --git a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Recommendations/schemas/recommendations_1.0.json b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Recommendations/schemas/recommendations_1.0.json new file mode 100644 index 000000000..5f296d7a3 --- /dev/null +++ b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Recommendations/schemas/recommendations_1.0.json @@ -0,0 +1,122 @@ +{ + "additionalColumns": [], + "translator": { + "type": "TabularTranslator", + "mappings": [ + { + "source": { + "path": "[['ResourceId']" + }, + "sink": { + "path": "ResourceId" + } + }, + { + "source": { + "path": "[['ResourceName']" + }, + "sink": { + "path": "ResourceName" + } + }, + { + "source": { + "path": "[['SubAccountId']" + }, + "sink": { + "path": "SubAccountId" + } + }, + { + "source": { + "path": "[['SubAccountName']" + }, + "sink": { + "path": "SubAccountName" + } + }, + { + "source": { + "path": "[['x_RecommendationCategory']" + }, + "sink": { + "path": "x_RecommendationCategory" + } + }, + { + "source": { + "path": "[['x_RecommendationDate']" + }, + "sink": { + "path": "x_RecommendationDate" + } + }, + { + "source": { + "path": "[['x_RecommendationDescription']" + }, + "sink": { + "path": "x_RecommendationDescription" + } + }, + { + "source": { + "path": "[['x_RecommendationDetails']" + }, + "sink": { + "path": "x_RecommendationDetails" + } + }, + { + "source": { + "path": "[['x_RecommendationId']" + }, + "sink": { + "path": "x_RecommendationId" + } + }, + { + "source": { + "path": "[['x_ResourceGroupName']" + }, + "sink": { + "path": "x_ResourceGroupName" + } + }, + { + "source": { + "path": "[['x_SourceName']" + }, + "sink": { + "path": "x_SourceName" + } + }, + { + "source": { + "path": "[['x_SourceProvider']" + }, + "sink": { + "path": "x_SourceProvider" + } + }, + { + "source": { + "path": "[['x_SourceType']" + }, + "sink": { + "path": "x_SourceType" + } + }, + { + "source": { + "path": "[['x_SourceVersion']" + }, + "sink": { + "path": "x_SourceVersion" + } + } + ], + "collectionReference": "$['data']", + "mapComplexValuesToString": true + } +} diff --git a/src/templates/finops-hub/modules/hub.bicep b/src/templates/finops-hub/modules/hub.bicep index af343e3ad..2b5603372 100644 --- a/src/templates/finops-hub/modules/hub.bicep +++ b/src/templates/finops-hub/modules/hub.bicep @@ -47,6 +47,15 @@ param remoteHubStorageKey string = '' @description('Optional. Enable managed exports where your FinOps hub instance will create and run Cost Management exports on your behalf. Not supported for Microsoft Customer Agreement (MCA) billing profiles. Requires the ability to grant User Access Administrator role to FinOps hubs, which is required to create Cost Management exports. Default: true.') param enableManagedExports bool = true +@description('Optional. Enable recommendations ingested from Azure Resource Graph based on configurable queries. The Data Factory managed identity requires Reader role on management groups or subscriptions to execute Resource Graph queries. Default: false.') +param enableRecommendations bool = false + +@description('Optional. Enable Azure Hybrid Benefit recommendations that flag VMs and SQL VMs without Azure Hybrid Benefit enabled. May generate noise if your organization does not have on-premises licenses. Requires enableRecommendations. Default: false.') +param enableAHBRecommendations bool = false + +@description('Optional. Enable non-Spot AKS cluster recommendations that flag AKS clusters with autoscaling but not using Spot VMs. May generate noise since Spot VMs are only appropriate for interruptible workloads. Requires enableRecommendations. Default: false.') +param enableSpotRecommendations bool = false + // cSpell:ignore eventhouse @description('Optional. Microsoft Fabric eventhouse query URI. Default: "" (do not use).') param fabricQueryUri string = '' @@ -301,6 +310,44 @@ module analytics 'Microsoft.FinOpsHubs/Analytics/app.bicep' = if (useFabric || u } } +//------------------------------------------------------------------------------ +// Ingestion queries +//------------------------------------------------------------------------------ + +module ingestionQueries 'Microsoft.FinOpsHubs/IngestionQueries/app.bicep' = if (enableRecommendations) { + name: 'Microsoft.FinOpsHubs.IngestionQueries' + params: { + app: newApp(hub, 'Microsoft.FinOpsHubs', 'IngestionQueries') + core: core.outputs.metadata + } +} + +module azureResourceGraph 'Microsoft.FinOpsHubs/AzureResourceGraph/app.bicep' = if (enableRecommendations) { + name: 'Microsoft.FinOpsHubs.AzureResourceGraph' + params: { + app: newApp(hub, 'Microsoft.FinOpsHubs', 'AzureResourceGraph') + core: core.outputs.metadata + } +} + +//------------------------------------------------------------------------------ +// Custom recommendations +//------------------------------------------------------------------------------ + +module recommendations 'Microsoft.FinOpsHubs/Recommendations/app.bicep' = if (enableRecommendations) { + name: 'Microsoft.FinOpsHubs.Recommendations' + dependsOn: [ + azureResourceGraph + ] + params: { + app: newApp(hub, 'Microsoft.FinOpsHubs', 'Recommendations') + core: core.outputs.metadata + ingestionQueries: ingestionQueries!.outputs.metadata // Safe: guarded by same enableRecommendations condition + enableAHBRecommendations: enableAHBRecommendations + enableSpotRecommendations: enableSpotRecommendations + } +} + //------------------------------------------------------------------------------ // Remote hub app //------------------------------------------------------------------------------ @@ -348,9 +395,10 @@ module startTriggers 'fx/hub-initialize.bicep' = { name: 'Microsoft.FinOpsHubs.StartTriggers' dependsOn: [ analytics + recommendations deleteOldResources remoteHub - cmManagedExports + cmManagedExports ] params: { app: core.outputs.app