piglet/run at github-release · maparco/piglet · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
#!/bin/bash

SIZE=10g
DATAFILE=hdfs:/data/edbt2016/observations${SIZE}.nt
#DATAFILE=hdfs:/data/edbt2016/observations10g.nt
#DATAFILE=hdfs:/data/edbt2016/obs100000.nt
#DATAFILE=file:/home/hage/Documents/uni/data/srbench/obs1000.nt
SENSORFILE=hdfs:/data/edbt2016/sensors.nt
GENBASEDIR=/home/hage/Documents/uni/stuff/edbt2016/${SIZE}/gen/
OUTDIR=hdfs:/data/edbt2016/${SIZE}
#OUTDIR=file:/tmp/
LOGDIR=/home/hage/Documents/uni/stuff/edbt2016/${SIZE}
TS_TEMP=800
TS_WIND=0
EXECUTORS=16
CORES=2
MASTER=yarn

#######################
#blizzard 1

echo "running blizzard 1 "

./script/pigs -o $GENBASEDIR -p file=$DATAFILE,outfile=$OUTDIR/blizzard1,sensorfile=$SENSORFILE,temp_threshold=$TS_TEMP,wind_threshold=$TS_WIND -m ${MASTER} --backend-args "--num-executors=${EXECUTORS},--executor-cores=${CORES}"  ~/Documents/uni/pub/edbt2016/queries/blizzard_1_plain.pig > $LOGDIR/blizzard1.log 2>&1;


#######################
#blizzard 2

echo "running blizzard 2"

#./script/pigs -o $GENBASEDIR -p file=$DATAFILE,outfile=$OUTDIR/blizzard2,sensorfile=$SENSORFILE -m ${MASTER} --backend-args "--num-executors=${EXECUTORS},--executor-cores=${CORES}"  ~/Documents/uni/pub/edbt2016/queries/blizzard_2_plain.pig > $LOGDIR/blizzard2.log 2>&1


#######################
#blizzard merge

echo "running blizzard merge "

#./script/pigs -o $GENBASEDIR -p file=$DATAFILE,outfile=$OUTDIR/blizzard_merge,outfile2=$OUTDIR/blizzard_merge2,temp_threshold=$TS_TEMP,wind_threshold=$TS_WIND,sensorfile=$SENSORFILE -m ${MASTER} --backend-args "--num-executors=${EXECUTORS},--executor-cores=${CORES}"    ~/Documents/uni/pub/edbt2016/queries/blizzard_merge.pig > $LOGDIR/blizzard_merge.log 2>&1


for i in $(seq 1 9);
do
  for j in 1 #2
  do
#      echo ""
    #echo "running blizzard $j store $i"
    #./script/pigs -o $GENBASEDIR -p file=$DATAFILE,outfile=$OUTDIR/blizzard${j}_store_$i,temp_threshold=$TS_TEMP,wind_threshold=$TS_WIND,store_file=$OUTDIR/blizzard${j}_mat_$i,sensorfile=$SENSORFILE -m ${MASTER} --backend-args "--num-executors=${EXECUTORS},--executor-cores=${CORES}"  ~/Documents/uni/pub/edbt2016/queries/blizzard_${j}_plain_store_$i.pig > $LOGDIR/blizzard${j}_store_$i.log 2>&1

    #echo "running blizzard $j load $i"

    #./script/pigs -o $GENBASEDIR -p file=$DATAFILE,outfile=$OUTDIR/blizzard${j}_load_$i,temp_threshold=$TS_TEMP,wind_threshold=$TS_WIND,store_file=$OUTDIR/blizzard${j}_mat_$i,sensorfile=$SENSORFILE -m ${MASTER} --backend-args "--num-executors=${EXECUTORS},--executor-cores=${CORES}"  ~/Documents/uni/pub/edbt2016/queries/blizzard_${j}_plain_load_$i.pig > $LOGDIR/blizzard${j}_load_$i.log 2>&1
  done
done