Skip to content

feat: support bf16 #124

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
yyihuang wants to merge 11 commits into
base: inference
Choose a base branch
from
Draft

feat: support bf16 #124

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
d7d52b0
init part1
yyihuang Apr 4, 2025
945a6d9
compile
yyihuang Apr 9, 2025
d4f403b
upd header
yyihuang Apr 9, 2025
8c8e48f
upd
yyihuang Apr 9, 2025
4ebf5d0
upd
yyihuang Apr 9, 2025
d7c9b9e
upd
yyihuang Apr 9, 2025
046d208
fix cublas type issue
yyihuang Apr 9, 2025
c628e1a
fix operator part1
yyihuang Apr 10, 2025
62f7e4a
fix operator part2
yyihuang Apr 10, 2025
4ffc43e
upd python peft interface
yyihuang Apr 10, 2025
3d6370e
tmp load bf16 from half-prec
yyihuang Apr 14, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 11 additions & 0 deletions include/flexflow/accessor.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,16 +5,25 @@

#if defined(FF_USE_CUDA)
#include <cuda_fp16.h>
#include <cuda_bf16.h>
#elif defined(FF_USE_HIP_CUDA)
#include <cuda_fp16.h>
#include <cuda_bf16.h>
#elif defined(FF_USE_HIP_ROCM)
#include <hip/hip_fp16.h>
#include <hip_bfloat16.h>
#endif

// using namespace Legion;

namespace FlexFlow {

#if defined(FF_USE_CUDA) || defined(FF_USE_HIP_CUDA)
typedef __nv_bfloat16 __ff_bfloat16;
#elif defined(FF_USE_HIP_ROCM)
typedef hip_bfloat16 __ff_bfloat16;
#endif

template <typename FT, int N, typename T = Legion::coord_t>
using AccessorRO =
Legion::FieldAccessor<READ_ONLY, FT, N, T, Realm::AffineAccessor<FT, N, T>>;
Expand Down Expand Up @@ -61,6 +70,7 @@ class GenericTensorAccessorW {
float *get_float_ptr() const;
double *get_double_ptr() const;
half *get_half_ptr() const;
__ff_bfloat16 *get_bfloat16_ptr() const;
char *get_byte_ptr() const;
DataType data_type;
Legion::Domain domain;
Expand All @@ -80,6 +90,7 @@ class GenericTensorAccessorR {
float const *get_float_ptr() const;
double const *get_double_ptr() const;
half const *get_half_ptr() const;
__ff_bfloat16 const *get_bfloat16_ptr() const;
char const *get_byte_ptr() const;
DataType data_type;
Legion::Domain domain;
Expand Down
9 changes: 8 additions & 1 deletion include/flexflow/ffconst.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ enum DataType {
DT_INT4 = 46,
DT_INT8 = 47,
DT_NONE = 49,
DT_BFLOAT16 = 50, // placeholder for bfloat16
DT_BFLOAT16 = 50,
};

enum LossType {
Expand Down Expand Up @@ -287,4 +287,11 @@ enum {
PEFT_MODEL_ID_FIRST_VALID = 6000000,
PEFT_MODEL_ID_LAST_VALID = 6999999
};

enum InferencePrecision {
INFERENCE_FLOAT = 800,
INFERENCE_HALF = 801,
INFERENCE_BFLOAT16 = 802,
};

#endif // _FLEXFLOW_CONST_H_
Loading
Loading