various fixes for scalable vectors

compiler/rustc_abi/src/layout.rs

@@ -10,8 +10,8 @@ use tracing::{debug, trace};
 
 use crate::{
     AbiAlign, Align, BackendRepr, FieldsShape, HasDataLayout, IndexSlice, IndexVec, Integer,
-    LayoutData, Niche, NonZeroUsize, Primitive, ReprOptions, Scalar, Size, StructKind, TagEncoding,
-    TargetDataLayout, Variants, WrappingRange,
+    LayoutData, Niche, NonZeroUsize, NumScalableVectors, Primitive, ReprOptions, Scalar, Size,
+    StructKind, TagEncoding, TargetDataLayout, Variants, WrappingRange,
 };
 
 mod coroutine;
@@ -204,13 +204,19 @@ impl<Cx: HasDataLayout> LayoutCalculator<Cx> {
         &self,
         element: F,
         count: u64,
+        number_of_vectors: NumScalableVectors,
     ) -> LayoutCalculatorResult<FieldIdx, VariantIdx, F>
     where
         FieldIdx: Idx,
         VariantIdx: Idx,
         F: AsRef<LayoutData<FieldIdx, VariantIdx>> + fmt::Debug,
     {
-        vector_type_layout(SimdVectorKind::Scalable, self.cx.data_layout(), element, count)
+        vector_type_layout(
+            SimdVectorKind::Scalable(number_of_vectors),
+            self.cx.data_layout(),
+            element,
+            count,
+        )
     }
 
     pub fn simd_type<FieldIdx, VariantIdx, F>(
@@ -1526,7 +1532,7 @@ impl<Cx: HasDataLayout> LayoutCalculator<Cx> {
 
 enum SimdVectorKind {
     /// `#[rustc_scalable_vector]`
-    Scalable,
+    Scalable(NumScalableVectors),
     /// `#[repr(simd, packed)]`
     PackedFixed,
     /// `#[repr(simd)]`
@@ -1559,9 +1565,10 @@ where
     let size =
         elt.size.checked_mul(count, dl).ok_or_else(|| LayoutCalculatorError::SizeOverflow)?;
     let (repr, align) = match kind {
-        SimdVectorKind::Scalable => {
-            (BackendRepr::SimdScalableVector { element, count }, dl.llvmlike_vector_align(size))
-        }
+        SimdVectorKind::Scalable(number_of_vectors) => (
+            BackendRepr::SimdScalableVector { element, count, number_of_vectors },
+            dl.llvmlike_vector_align(size),
+        ),
         // Non-power-of-two vectors have padding up to the next power-of-two.
         // If we're a packed repr, remove the padding while keeping the alignment as close
         // to a vector as possible.

compiler/rustc_abi/src/lib.rs

@@ -1702,6 +1702,28 @@ impl AddressSpace {
     pub const ZERO: Self = AddressSpace(0);
 }
 
+/// How many scalable vectors are in a `BackendRepr::ScalableVector`?
+#[derive(Clone, Copy, PartialEq, Eq, Hash, Debug)]
+#[cfg_attr(feature = "nightly", derive(HashStable_Generic))]
+pub struct NumScalableVectors(pub u8);
+
+impl NumScalableVectors {
+    /// Returns a `NumScalableVector` for a non-tuple scalable vector (e.g. a single vector).
+    pub fn for_non_tuple() -> Self {
+        NumScalableVectors(1)
+    }
+
+    // Returns `NumScalableVectors` for values of two through eight, which are a valid number of
+    // fields for a tuple of scalable vectors to have. `1` is a valid value of `NumScalableVectors`
+    // but not for a tuple which would have a field count.
+    pub fn from_field_count(count: usize) -> Option<Self> {
+        match count {
+            2..8 => Some(NumScalableVectors(count as u8)),
+            _ => None,
+        }
+    }
+}
+
 /// The way we represent values to the backend
 ///
 /// Previously this was conflated with the "ABI" a type is given, as in the platform-specific ABI.
@@ -1720,6 +1742,7 @@ pub enum BackendRepr {
     SimdScalableVector {
         element: Scalar,
         count: u64,
+        number_of_vectors: NumScalableVectors,
     },
     SimdVector {
         element: Scalar,
@@ -1826,8 +1849,12 @@ impl BackendRepr {
                 BackendRepr::SimdVector { element: element.to_union(), count }
             }
             BackendRepr::Memory { .. } => BackendRepr::Memory { sized: true },
-            BackendRepr::SimdScalableVector { element, count } => {
-                BackendRepr::SimdScalableVector { element: element.to_union(), count }
+            BackendRepr::SimdScalableVector { element, count, number_of_vectors } => {
+                BackendRepr::SimdScalableVector {
+                    element: element.to_union(),
+                    count,
+                    number_of_vectors,
+                }
             }
         }
     }
@@ -2167,7 +2194,7 @@ impl<FieldIdx: Idx, VariantIdx: Idx> LayoutData<FieldIdx, VariantIdx> {
     }
 
     /// Returns `true` if the size of the type is only known at runtime.
-    pub fn is_runtime_sized(&self) -> bool {
+    pub fn is_scalable_vector(&self) -> bool {
         matches!(self.backend_repr, BackendRepr::SimdScalableVector { .. })
     }
 

compiler/rustc_codegen_gcc/src/builder.rs

@@ -24,7 +24,8 @@ use rustc_data_structures::fx::FxHashSet;
 use rustc_middle::bug;
 use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrs;
 use rustc_middle::ty::layout::{
-    FnAbiError, FnAbiOfHelpers, FnAbiRequest, HasTyCtxt, HasTypingEnv, LayoutError, LayoutOfHelpers,
+    FnAbiError, FnAbiOfHelpers, FnAbiRequest, HasTyCtxt, HasTypingEnv, LayoutError,
+    LayoutOfHelpers, TyAndLayout,
 };
 use rustc_middle::ty::{self, AtomicOrdering, Instance, Ty, TyCtxt};
 use rustc_span::Span;
@@ -943,8 +944,8 @@ impl<'a, 'gcc, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'gcc, 'tcx> {
             .get_address(self.location)
     }
 
-    fn scalable_alloca(&mut self, _elt: u64, _align: Align, _element_ty: Ty<'_>) -> RValue<'gcc> {
-        todo!()
+    fn alloca_with_ty(&mut self, ty: TyAndLayout<'tcx>) -> RValue<'gcc> {
+        self.alloca(ty.layout.size, ty.layout.align.abi)
     }
 
     fn load(&mut self, pointee_ty: Type<'gcc>, ptr: RValue<'gcc>, align: Align) -> RValue<'gcc> {

compiler/rustc_codegen_gcc/src/common.rs

@@ -145,6 +145,10 @@ impl<'gcc, 'tcx> ConstCodegenMethods for CodegenCx<'gcc, 'tcx> {
         self.const_int(self.type_i32(), i as i64)
     }
 
+    fn const_i64(&self, i: i64) -> RValue<'gcc> {
+        self.const_int(self.type_i64(), i)
+    }
+
     fn const_int(&self, typ: Type<'gcc>, int: i64) -> RValue<'gcc> {
         self.gcc_int(typ, int)
     }

compiler/rustc_codegen_llvm/src/builder.rs

@@ -7,8 +7,7 @@ pub(crate) mod autodiff;
 pub(crate) mod gpu_offload;
 
 use libc::{c_char, c_uint};
-use rustc_abi as abi;
-use rustc_abi::{Align, Size, WrappingRange};
+use rustc_abi::{self as abi, Align, Size, WrappingRange};
 use rustc_codegen_ssa::MemFlags;
 use rustc_codegen_ssa::common::{IntPredicate, RealPredicate, SynchronizationScope, TypeKind};
 use rustc_codegen_ssa::mir::operand::{OperandRef, OperandValue};
@@ -616,21 +615,14 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
         }
     }
 
-    fn scalable_alloca(&mut self, elt: u64, align: Align, element_ty: Ty<'_>) -> Self::Value {
+    fn alloca_with_ty(&mut self, layout: TyAndLayout<'tcx>) -> Self::Value {
         let mut bx = Builder::with_cx(self.cx);
         bx.position_at_start(unsafe { llvm::LLVMGetFirstBasicBlock(self.llfn()) });
-        let llvm_ty = match element_ty.kind() {
-            ty::Bool => bx.type_i1(),
-            ty::Int(int_ty) => self.cx.type_int_from_ty(*int_ty),
-            ty::Uint(uint_ty) => self.cx.type_uint_from_ty(*uint_ty),
-            ty::Float(float_ty) => self.cx.type_float_from_ty(*float_ty),
-            _ => unreachable!("scalable vectors can only contain a bool, int, uint or float"),
-        };
+        let scalable_vector_ty = layout.llvm_type(self.cx);
 
         unsafe {
-            let ty = llvm::LLVMScalableVectorType(llvm_ty, elt.try_into().unwrap());
-            let alloca = llvm::LLVMBuildAlloca(&bx.llbuilder, ty, UNNAMED);
-            llvm::LLVMSetAlignment(alloca, align.bytes() as c_uint);
+            let alloca = llvm::LLVMBuildAlloca(&bx.llbuilder, scalable_vector_ty, UNNAMED);
+            llvm::LLVMSetAlignment(alloca, layout.align.abi.bytes() as c_uint);
             alloca
         }
     }

compiler/rustc_codegen_llvm/src/common.rs

@@ -159,6 +159,10 @@ impl<'ll, 'tcx> ConstCodegenMethods for CodegenCx<'ll, 'tcx> {
         self.const_int(self.type_i32(), i as i64)
     }
 
+    fn const_i64(&self, i: i64) -> &'ll Value {
+        self.const_int(self.type_i64(), i as i64)
+    }
+
     fn const_int(&self, t: &'ll Type, i: i64) -> &'ll Value {
         debug_assert!(
             self.type_kind(t) == TypeKind::Integer,

compiler/rustc_codegen_llvm/src/debuginfo/dwarf_const.rs

@@ -35,6 +35,14 @@ declare_constant!(DW_OP_plus_uconst: u64);
 /// Double-checked by a static assertion in `RustWrapper.cpp`.
 #[allow(non_upper_case_globals)]
 pub(crate) const DW_OP_LLVM_fragment: u64 = 0x1000;
+#[allow(non_upper_case_globals)]
+pub(crate) const DW_OP_constu: u64 = 0x10;
+#[allow(non_upper_case_globals)]
+pub(crate) const DW_OP_minus: u64 = 0x1c;
+#[allow(non_upper_case_globals)]
+pub(crate) const DW_OP_mul: u64 = 0x1e;
+#[allow(non_upper_case_globals)]
+pub(crate) const DW_OP_bregx: u64 = 0x92;
 // It describes the actual value of a source variable which might not exist in registers or in memory.
 #[allow(non_upper_case_globals)]
 pub(crate) const DW_OP_stack_value: u64 = 0x9f;

compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs

@@ -3,10 +3,10 @@ use std::fmt::{self, Write};
 use std::hash::{Hash, Hasher};
 use std::path::PathBuf;
 use std::sync::Arc;
-use std::{iter, ptr};
+use std::{assert_matches, iter, ptr};
 
 use libc::{c_longlong, c_uint};
-use rustc_abi::{Align, Size};
+use rustc_abi::{Align, Layout, NumScalableVectors, Size};
 use rustc_codegen_ssa::debuginfo::type_names::{VTableNameKind, cpp_like_debuginfo};
 use rustc_codegen_ssa::traits::*;
 use rustc_hir::def::{CtorKind, DefKind};
@@ -16,12 +16,12 @@ use rustc_middle::ty::layout::{
     HasTypingEnv, LayoutOf, TyAndLayout, WIDE_PTR_ADDR, WIDE_PTR_EXTRA,
 };
 use rustc_middle::ty::{
-    self, AdtKind, CoroutineArgsExt, ExistentialTraitRef, Instance, Ty, TyCtxt, Visibility,
+    self, AdtDef, AdtKind, CoroutineArgsExt, ExistentialTraitRef, Instance, Ty, TyCtxt, Visibility,
 };
 use rustc_session::config::{self, DebugInfo, Lto};
 use rustc_span::{DUMMY_SP, FileName, RemapPathScopeComponents, SourceFile, Span, Symbol, hygiene};
 use rustc_symbol_mangling::typeid_for_trait_ref;
-use rustc_target::spec::DebuginfoKind;
+use rustc_target::spec::{Arch, DebuginfoKind};
 use smallvec::smallvec;
 use tracing::{debug, instrument};
 
@@ -33,7 +33,7 @@ use super::type_names::{compute_debuginfo_type_name, compute_debuginfo_vtable_na
 use super::utils::{DIB, debug_context, get_namespace_for_item, is_node_local_to_unit};
 use crate::common::{AsCCharPtr, CodegenCx};
 use crate::debuginfo::metadata::type_map::build_type_with_children;
-use crate::debuginfo::utils::{WidePtrKind, wide_pointer_kind};
+use crate::debuginfo::utils::{WidePtrKind, create_DIArray, wide_pointer_kind};
 use crate::debuginfo::{DIBuilderExt, dwarf_const};
 use crate::llvm::debuginfo::{
     DIBasicType, DIBuilder, DICompositeType, DIDescriptor, DIFile, DIFlags, DILexicalBlock,
@@ -1039,6 +1039,7 @@ fn build_struct_type_di_node<'ll, 'tcx>(
     span: Span,
 ) -> DINodeCreationResult<'ll> {
     let struct_type = unique_type_id.expect_ty();
+
     let ty::Adt(adt_def, _) = struct_type.kind() else {
         bug!("build_struct_type_di_node() called with non-struct-type: {:?}", struct_type);
     };
@@ -1051,14 +1052,29 @@ fn build_struct_type_di_node<'ll, 'tcx>(
     } else {
         None
     };
+    let name = compute_debuginfo_type_name(cx.tcx, struct_type, false);
+
+    if struct_type.is_scalable_vector() {
+        let parts = struct_type.scalable_vector_parts(cx.tcx).unwrap();
+        return build_scalable_vector_di_node(
+            cx,
+            unique_type_id,
+            name,
+            *adt_def,
+            parts,
+            struct_type_and_layout.layout,
+            def_location,
+            containing_scope,
+        );
+    }
 
     type_map::build_type_with_children(
         cx,
         type_map::stub(
             cx,
             Stub::Struct,
             unique_type_id,
-            &compute_debuginfo_type_name(cx.tcx, struct_type, false),
+            &name,
             def_location,
             size_and_align_of(struct_type_and_layout),
             Some(containing_scope),
@@ -1101,6 +1117,100 @@ fn build_struct_type_di_node<'ll, 'tcx>(
     )
 }
 
+/// Generate debuginfo for a `#[rustc_scalable_vector]` type.
+///
+/// Debuginfo for a scalable vector uses a derived type based on a composite type. The composite
+/// type has the  `DIFlagVector` flag set and is based on the element type of the scalable vector.
+/// The composite type has a subrange from 0 to an expression that calculates the number of
+/// elements in the vector.
+///
+/// ```text,ignore
+/// !1 = !DIDerivedType(tag: DW_TAG_typedef, name: "svint16_t", ..., baseType: !2, ...)
+/// !2 = !DICompositeType(tag: DW_TAG_array_type, baseType: !3, ..., flags: DIFlagVector, elements: !4)
+/// !3 = !DIBasicType(name: "i16", size: 16, encoding: DW_ATE_signed)
+/// !4 = !{!5}
+/// !5 = !DISubrange(lowerBound: 0, upperBound: !DIExpression(DW_OP_constu, 4, DW_OP_bregx, 46, 0, DW_OP_mul, DW_OP_constu, 1, DW_OP_minus))
+/// ```
+///
+/// See the `CodegenType::CreateType(const BuiltinType *BT)` implementation in Clang for how this
+/// is generated for C and C++.
+fn build_scalable_vector_di_node<'ll, 'tcx>(
+    cx: &CodegenCx<'ll, 'tcx>,
+    unique_type_id: UniqueTypeId<'tcx>,
+    name: String,
+    adt_def: AdtDef<'tcx>,
+    (element_count, element_ty, number_of_vectors): (u16, Ty<'tcx>, NumScalableVectors),
+    layout: Layout<'tcx>,
+    def_location: Option<DefinitionLocation<'ll>>,
+    containing_scope: &'ll DIScope,
+) -> DINodeCreationResult<'ll> {
+    use dwarf_const::{DW_OP_bregx, DW_OP_constu, DW_OP_minus, DW_OP_mul};
+    assert!(adt_def.repr().scalable());
+    // This logic is specific to AArch64 for the moment, but can be extended for other architectures
+    // later.
+    assert_matches!(cx.tcx.sess.target.arch, Arch::AArch64);
+
+    let (file_metadata, line_number) = if let Some(def_location) = def_location {
+        (def_location.0, def_location.1)
+    } else {
+        (unknown_file_metadata(cx), UNKNOWN_LINE_NUMBER)
+    };
+
+    let (bitstride, element_di_node) = if element_ty.is_bool() {
+        (Some(llvm::LLVMValueAsMetadata(cx.const_i64(1))), type_di_node(cx, cx.tcx.types.u8))
+    } else {
+        (None, type_di_node(cx, element_ty))
+    };
+
+    let number_of_elements: u64 = (element_count as u64) * (number_of_vectors.0 as u64);
+    let number_of_elements_per_vg = number_of_elements / 2;
+    let mut expr = smallvec::SmallVec::<[u64; 9]>::new();
+    // `($number_of_elements_per_vector_granule * (value_of_register(AArch64::VG) + 0)) - 1`
+    expr.push(DW_OP_constu); // Push a constant onto the stack
+    expr.push(number_of_elements_per_vg);
+    expr.push(DW_OP_bregx); // Push the value of a register + offset on to the stack
+    expr.push(/* AArch64::VG */ 46u64);
+    expr.push(0u64);
+    expr.push(DW_OP_mul); // Multiply top two values on stack
+    expr.push(DW_OP_constu); // Push a constant onto the stack
+    expr.push(1u64);
+    expr.push(DW_OP_minus); // Subtract top two values on stack
+
+    let di_builder = DIB(cx);
+    let metadata = unsafe {
+        let upper = llvm::LLVMDIBuilderCreateExpression(di_builder, expr.as_ptr(), expr.len());
+        let subrange = llvm::LLVMRustDIGetOrCreateSubrange(
+            di_builder,
+            /* CountNode */ None,
+            llvm::LLVMValueAsMetadata(cx.const_i64(0)),
+            upper,
+            /* Stride */ None,
+        );
+        let subscripts = create_DIArray(di_builder, &[Some(subrange)]);
+        let vector_ty = llvm::LLVMRustDICreateVectorType(
+            di_builder,
+            /* Size */ 0,
+            layout.align.bits() as u32,
+            element_di_node,
+            subscripts,
+            bitstride,
+        );
+        llvm::LLVMDIBuilderCreateTypedef(
+            di_builder,
+            vector_ty,
+            name.as_ptr(),
+            name.len(),
+            file_metadata,
+            line_number,
+            Some(containing_scope),
+            layout.align.bits() as u32,
+        )
+    };
+
+    debug_context(cx).type_map.insert(unique_type_id, metadata);
+    DINodeCreationResult { di_node: metadata, already_stored_in_typemap: true }
+}
+
 //=-----------------------------------------------------------------------------
 // Tuples
 //=-----------------------------------------------------------------------------