diff --git a/models/DAMO-NLP-MT/polylm-1.7b/harness.sh b/models/DAMO-NLP-MT/polylm-1.7b/harness.sh new file mode 100644 index 00000000000..a20132397db --- /dev/null +++ b/models/DAMO-NLP-MT/polylm-1.7b/harness.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +PRETRAINED="DAMO-NLP-MT/polylm-1.7b" +TOKENIZER="DAMO-NLP-MT/polylm-1.7b,use_fast=False" +MODEL_ARGS="pretrained=${PRETRAINED},tokenizer=${TOKENIZER}" +TASK="jcommonsenseqa-1.1-0.2,jnli-1.1-0.2,marc_ja-1.1-0.2,jsquad-1.1-0.2,jaqket_v2-0.1-0.2,xlsum_ja,xwinograd_ja,mgsm" +NUM_FEWSHOT="3,3,3,2,1,1,0,5" +OUTPUT_PATH="models/DAMO-NLP-MT/polylm-1.7b/result.json" +python main.py \ + --model hf-causal \ + --model_args $MODEL_ARGS \ + --tasks $TASK \ + --num_fewshot $NUM_FEWSHOT \ + --device "cuda" \ + --no_cache \ + --output_path $OUTPUT_PATH \ No newline at end of file diff --git a/models/DAMO-NLP-MT/polylm-1.7b/result.json b/models/DAMO-NLP-MT/polylm-1.7b/result.json new file mode 100644 index 00000000000..49d600c2b96 --- /dev/null +++ b/models/DAMO-NLP-MT/polylm-1.7b/result.json @@ -0,0 +1,71 @@ +{ + "results": { + "jcommonsenseqa-1.1-0.2": { + "acc": 0.27882037533512066, + "acc_stderr": 0.013411059090653923, + "acc_norm": 0.24664879356568364, + "acc_norm_stderr": 0.012891912227940063 + }, + "jnli-1.1-0.2": { + "acc": 0.5168447000821693, + "acc_stderr": 0.010131000879683306, + "acc_norm": 0.33853738701725555, + "acc_norm_stderr": 0.009593673089127825 + }, + "marc_ja-1.1-0.2": { + "acc": 0.6731517509727627, + "acc_stderr": 0.006238642199187777, + "acc_norm": 0.6731517509727627, + "acc_norm_stderr": 0.006238642199187777 + }, + "xwinograd_ja": { + "acc": 0.5401459854014599, + "acc_stderr": 0.01610211088018111 + }, + "jsquad-1.1-0.2": { + "exact_match": 20.171094101755966, + "f1": 35.56702411988748 + }, + "jaqket_v2-0.1-0.2": { + "exact_match": 4.982817869415808, + "f1": 12.956318214050183 + }, + "xlsum_ja": { + "rouge2": 7.208548805942722 + }, + "mgsm": { + "acc": 0.012, + "acc_stderr": 0.006900323023694271 + } + }, + "versions": { + "jcommonsenseqa-1.1-0.2": 1.1, + "jnli-1.1-0.2": 1.1, + "marc_ja-1.1-0.2": 1.1, + "jsquad-1.1-0.2": 1.1, + "jaqket_v2-0.1-0.2": 0.1, + "xlsum_ja": 1.0, + "xwinograd_ja": 1.0, + "mgsm": 1.0 + }, + "config": { + "model": "hf-causal", + "model_args": "pretrained=DAMO-NLP-MT/polylm-1.7b,tokenizer=DAMO-NLP-MT/polylm-1.7b,use_fast=False", + "num_fewshot": [ + 3, + 3, + 3, + 2, + 1, + 1, + 0, + 5 + ], + "batch_size": null, + "device": "cuda", + "no_cache": true, + "limit": null, + "bootstrap_iters": 100000, + "description_dict": {} + } +} \ No newline at end of file diff --git a/models/DAMO-NLP-MT/polylm-13b/harness.sh b/models/DAMO-NLP-MT/polylm-13b/harness.sh new file mode 100644 index 00000000000..3e2685c0bfb --- /dev/null +++ b/models/DAMO-NLP-MT/polylm-13b/harness.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +PRETRAINED="DAMO-NLP-MT/polylm-13b" +TOKENIZER="DAMO-NLP-MT/polylm-13b,use_fast=False" +MODEL_ARGS="pretrained=${PRETRAINED},tokenizer=${TOKENIZER}" +TASK="jcommonsenseqa-1.1-0.2,jnli-1.1-0.2,marc_ja-1.1-0.2,jsquad-1.1-0.2,jaqket_v2-0.1-0.2,xlsum_ja,xwinograd_ja,mgsm" +NUM_FEWSHOT="3,3,3,2,1,1,0,5" +OUTPUT_PATH="models/DAMO-NLP-MT/polylm-13b/result.json" +python main.py \ + --model hf-causal \ + --model_args $MODEL_ARGS \ + --tasks $TASK \ + --num_fewshot $NUM_FEWSHOT \ + --device "cuda" \ + --no_cache \ + --output_path $OUTPUT_PATH \ No newline at end of file diff --git a/models/DAMO-NLP-MT/polylm-13b/result.json b/models/DAMO-NLP-MT/polylm-13b/result.json new file mode 100644 index 00000000000..a68a5eafce0 --- /dev/null +++ b/models/DAMO-NLP-MT/polylm-13b/result.json @@ -0,0 +1,71 @@ +{ + "results": { + "jcommonsenseqa-1.1-0.2": { + "acc": 0.3297587131367292, + "acc_stderr": 0.014060242997081397, + "acc_norm": 0.2529043789097408, + "acc_norm_stderr": 0.013000060342436679 + }, + "jnli-1.1-0.2": { + "acc": 0.5534100246507806, + "acc_stderr": 0.010078756236647866, + "acc_norm": 0.4926047658175842, + "acc_norm_stderr": 0.01013564618952122 + }, + "marc_ja-1.1-0.2": { + "acc": 0.42465511142553947, + "acc_stderr": 0.006574198527911998, + "acc_norm": 0.42465511142553947, + "acc_norm_stderr": 0.006574198527911998 + }, + "xwinograd_ja": { + "acc": 0.662148070907195, + "acc_stderr": 0.015281219220241183 + }, + "jsquad-1.1-0.2": { + "exact_match": 70.41873030166592, + "f1": 82.58920858925727 + }, + "jaqket_v2-0.1-0.2": { + "exact_match": 53.350515463917525, + "f1": 60.14697168047684 + }, + "xlsum_ja": { + "rouge2": 15.0648799446579 + }, + "mgsm": { + "acc": 0.02, + "acc_stderr": 0.008872139507342683 + } + }, + "versions": { + "jcommonsenseqa-1.1-0.2": 1.1, + "jnli-1.1-0.2": 1.1, + "marc_ja-1.1-0.2": 1.1, + "jsquad-1.1-0.2": 1.1, + "jaqket_v2-0.1-0.2": 0.1, + "xlsum_ja": 1.0, + "xwinograd_ja": 1.0, + "mgsm": 1.0 + }, + "config": { + "model": "hf-causal", + "model_args": "pretrained=DAMO-NLP-MT/polylm-13b,tokenizer=DAMO-NLP-MT/polylm-13b,use_fast=False", + "num_fewshot": [ + 3, + 3, + 3, + 2, + 1, + 1, + 0, + 5 + ], + "batch_size": null, + "device": "cuda", + "no_cache": true, + "limit": null, + "bootstrap_iters": 100000, + "description_dict": {} + } +} \ No newline at end of file