diff --git a/results/analysis.R b/results/analysis.R index d2baea7..2a946fb 100644 --- a/results/analysis.R +++ b/results/analysis.R @@ -1,7 +1,9 @@ library("tidyverse") library("knitr") -data=read_csv("results.csv") +data=read_csv("results.csv") +ccgrid=read_csv("ccgrid.csv") + r_=function(x){round(x,digits=2)} @@ -30,11 +32,32 @@ receiverUptime=data%>%filter(isSender==0)%>%group_by(simkey,wireless,wakeupfor)% finalStats=finalStats%>%left_join(senderUptime,by=c("simkey","wireless","wakeupfor")) finalStats=finalStats%>%left_join(receiverUptime,by=c("simkey","wireless","wakeupfor")) +# Comparison +finalStats=finalStats%>%left_join(ccgrid,by=c("simkey","wireless","wakeupfor"),suffix = c("","_ccgrid")) +finalStats=finalStats%>%mutate(senderEnergy_err=abs(r_((senderEnergy-senderEnergy_ccgrid)/senderEnergy)*100)) +finalStats=finalStats%>%mutate(receiverEnergy_err=abs(r_((receiverEnergy-receiverEnergy_ccgrid)/receiverEnergy)*100)) +finalStats=finalStats%>%mutate(senderUptime_err=abs(r_((senderUptime-senderUptime_ccgrid)/senderUptime)*100)) +finalStats=finalStats%>%mutate(receiverUptime_err=abs(r_((receiverUptime-receiverUptime_ccgrid)/receiverUptime)*100)) # Add nSendFail to stats sendFail=data%>%filter(node=="on0")%>%group_by(simkey,wireless,wakeupfor)%>%summarise(nSendFail=mean(nSendFail)) finalStats=finalStats%>%left_join(sendFail,by=c("simkey","wireless","wakeupfor")) + +ggplot(finalStats,aes(x=simkey,senderEnergy_err,color=wireless,group=wireless))+geom_point()+geom_line()+facet_wrap(~wakeupfor)+xlab("Scenarios") + ylab("Sender Energy Relative Error (%)") + expand_limits(x = 0, y = 0) +ggsave("err_sender_energy.png",width=7,height = 5) + +ggplot(finalStats,aes(x=simkey,receiverEnergy_err,color=wireless,group=wireless))+geom_point()+geom_line()+facet_wrap(~wakeupfor)+xlab("Scenarios") + ylab("Receiver Energy Relative Error (%)") + expand_limits(x = 0, y = 0) +ggsave("err_receiver_energy.png",width=7,height = 5) + +ggplot(finalStats,aes(x=simkey,senderUptime_err,color=wireless,group=wireless))+geom_point()+geom_line()+facet_wrap(~wakeupfor)+xlab("Scenarios") + ylab("Sender AccUptime Energy Relative Error (%)") + expand_limits(x = 0, y = 0) +ggsave("err_sender_accuptime.png",width=7,height = 5) + +ggplot(finalStats,aes(x=simkey,receiverUptime_err,color=wireless,group=wireless))+geom_point()+geom_line()+facet_wrap(~wakeupfor)+xlab("Scenarios") + ylab("Receiver AccUptime Energy Relative Error (%)") + expand_limits(x = 0, y = 0) +ggsave("err_rcv_accuptime.png",width=7,height = 5) + + +stopifnot(1) # Generate plots ggplot(finalEnergy,aes(x=simkey,y=energy,fill=wireless))+ geom_bar(stat="identity",position=position_dodge())+ @@ -57,6 +80,7 @@ ggplot(finalStats,aes(x=simkey,y=success,fill=wireless))+ facet_wrap(~ wakeupfor) + ggsave("success.png") +##### Save tables ##### tf="tables-energy.org" write("",file=tf) for(wake in unique(data$wakeupfor)){ diff --git a/results/ccgrid.csv b/results/ccgrid.csv new file mode 100644 index 0000000..b31148e --- /dev/null +++ b/results/ccgrid.csv @@ -0,0 +1,17 @@ +simkey,wireless,wakeupfor,sd_success,success,senderEnergy,receiverEnergy,senderUptime,receiverUptime +baseline,lora,60,0,0,619.73,579.84,1440,1440 +extended,lora,60,2,7.70,1207.61,627.35,2564.53,1523.52 +hint,lora,60,0,0,753.42,957.64,1440,2354.94 +hintandextended,lora,60,2.20,9.80,1397.98,663.58,2904.50,1602.39 +baseline,nbiot,60,1.12,1.27,749.69,591.07,1440,1440 +extended,nbiot,60,1.80,7.16,946.39,599.13,1788.32,1447.54 +hint,nbiot,60,3.45,9.72,1007.06,702.59,1440,1666.51 +hintandextended,nbiot,60,2.36,9.41,1066.66,623.30,1901.47,1492.53 +baseline,lora,180,1.40,1.91,2026.23,1755.92,4320,4320 +extended,lora,180,0.70,11.45,2692.46,1772.35,6038.78,4364.27 +hint,lora,180,0,12, 2094.09,2169.50,4320,5344.86 +hintandextended,lora,180,0.24,11.94,2757.79,1867.34,6155.50,4600.09 +baseline,nbiot,180,1.33,9.90,2117.65,1761.78,4320,4320 +extended,nbiot,180,0.87,11.23,2311.52,1760.92,4868.78,4323.71 +hint,nbiot,180,0.37,11.92,2180.07,1868.54,4320,4576.83 +hintandextended,nbiot,180,0.41,11.88,2151.14,1822.44,4906.10,4473.32 diff --git a/results/energy.png b/results/energy.png index ebae2f0..7dfe834 100644 Binary files a/results/energy.png and b/results/energy.png differ diff --git a/results/err_rcv_accuptime.png b/results/err_rcv_accuptime.png new file mode 100644 index 0000000..35a95c1 Binary files /dev/null and b/results/err_rcv_accuptime.png differ diff --git a/results/err_receiver_energy.png b/results/err_receiver_energy.png new file mode 100644 index 0000000..7649448 Binary files /dev/null and b/results/err_receiver_energy.png differ diff --git a/results/err_sender_accuptime.png b/results/err_sender_accuptime.png new file mode 100644 index 0000000..93259d1 Binary files /dev/null and b/results/err_sender_accuptime.png differ diff --git a/results/err_sender_energy.png b/results/err_sender_energy.png new file mode 100644 index 0000000..b0f2a65 Binary files /dev/null and b/results/err_sender_energy.png differ diff --git a/results/paper_.sh b/results/paper_.sh new file mode 100755 index 0000000..c87543a --- /dev/null +++ b/results/paper_.sh @@ -0,0 +1,139 @@ +#!/usr/bin/env bash + +set -e + +wai=$(dirname $(readlink -f "$0")) +scenarios="${wai}/../scenarios" +inputs="${wai}/../inputs.json" +simulator="make -C ${wai}/../ run" +parser="${wai}/../parser.awk" +results="${wai}/results_.csv" +aheaders="simkey,wireless,wakeupfor" +avalues="none,none,none" +log_file="${wai}/logs/$(date +%s)_.org" && mkdir -p "${wai}/logs/" +gen_log=1 # Should we generate logs ? + +run-simulation () { + # Generate inputs + $scenarios $seed $simtime $wakeupevery $wakeupfor $n_nodes $extended $hint $poff $pon $prx $ptx $datasize $bitrate $hintsize $latency > "$inputs" + + # Init logs + [ $gen_log -eq 1 ] && echo -e "* seed=$seed simtime=$simtime wakeupevery=$wakeupevery wakeupfor=$wakeupfor n_nodes=$n_nodes extended=$extended hint=$hint poff=$poff pon=$pon prx=$prx ptx=$ptx datasize=$datasize bitrate=$bitrate \n" >> "${log_file}" + + # Run simulations + if [ -e "$results" ] + then + # Create new result file + if [ $gen_log -eq 1 ] + then + $simulator 2>&1 | tee -a "${log_file}" | $parser | sed 1d | sed "s/$/,${avalues}/" >> "$results" + else + $simulator 2>&1 | $parser | sed 1d | sed "s/$/,${avalues}/" >> "$results" + fi + else + # Use existing result file + if [ $gen_log -eq 1 ] + then + $simulator 2>&1 | tee -a "${log_file}" | $parser | sed "1 s/$/,${aheaders}/g" | sed "2,\$s/$/,${avalues}/" > "$results" + else + $simulator 2>&1 | $parser | sed "1 s/$/,${aheaders}/g" | sed "2,\$s/$/,${avalues}/" > "$results" + fi + fi +} + +# Default Parameters +seed=1 +simtime=86400 # One day +wakeupevery=3600 +wakeupfor=60 +n_nodes=12 +extended="false" +hint="false" +poff=0 +pon=0.4 +prx=0.16 +ptx=0.16 +datasize=1000000 # 1Mb +hintsize=8 # Integer +latency=0 # in Seconds +bitrate="100kbps" + + + +run-scenarios() { + # Configure number of seed per scenarios + nseed=1 + + # Baseline + avalues="baseline,$wireless,$wakeupfor" + for seed in $(seq 1 $nseed) + do + printf "\rBaseline...${seed}" + #run-simulation + done + echo + + # Hint + hint="true" + avalues="hint,$wireless,$wakeupfor" + for seed in $(seq 1 $nseed) + do + printf "\rHint...${seed}" + run-simulation + done + hint="false" + echo + + # Extended + extended="true" + avalues="extended,$wireless,$wakeupfor" + for seed in $(seq 1 $nseed) + do + printf "\rExtended...${seed}" + #run-simulation + done + extended="false" + echo + + # Hint+Extended + extended="true" + hint="true" + avalues="hintandextended,$wireless,$wakeupfor" + for seed in $(seq 1 $nseed) + do + printf "\rHint + Extended...${seed}" + #run-simulation + done + extended="false" + hint="false" + echo +} + +# Clean previous runs +[ -e "${results}" ] && rm "${results}" + +for wakeupfor in 60 +do + + # Lora + echo "----- Run Lora (wakeupfor=$wakeupfor) -----" + wireless="lora" + bitrate="50kbps" + pon=0.4 + prx=0.16 + ptx=0.16 + latency=0 + #run-scenarios + + # NbIot + echo "----- Run NbIoT (wakeupfor=$wakeupfor) -----" + wireless="nbiot" + bitrate="200kbps" + pon=0.4 + prx=0.65 + ptx=0.65 + latency=0.01 + run-scenarios +done + +echo "Logs: $log_file" \ No newline at end of file diff --git a/results/results_.csv b/results/results_.csv new file mode 100644 index 0000000..1e98970 --- /dev/null +++ b/results/results_.csv @@ -0,0 +1,13 @@ +node,isSender,nSend,nWakeUp,nDataRcv,nSendFail,nRcvFail,totalUptime,seed,hint_added,energy,simkey,wireless,wakeupfor +on6,0,0,24,0,0,1428,1440.000000,1,0,584.450000,hint,nbiot,60 +on9,0,0,24,0,0,1440,1440.000000,1,0,576.000000,hint,nbiot,60 +on11,0,0,24,0,0,1420,1440.000000,1,0,589.650000,hint,nbiot,60 +on8,0,0,24,1,0,439,1441.989680,1,0,606.052580,hint,nbiot,60 +on2,0,0,24,1,0,483,1440.010320,1,0,602.010836,hint,nbiot,60 +on10,0,0,24,0,0,1440,1440.000000,1,0,576.000000,hint,nbiot,60 +on5,0,0,24,0,0,1431,1440.000000,1,0,582.500000,hint,nbiot,60 +on3,0,0,24,0,0,1440,1440.000000,1,0,576.000000,hint,nbiot,60 +on4,0,0,24,0,0,1440,1440.000000,1,0,576.000000,hint,nbiot,60 +on1,0,0,24,0,0,1440,1440.000000,1,0,576.000000,hint,nbiot,60 +on7,0,0,24,1,0,282,1494.979360,1,1,636.160160,hint,nbiot,60 +on0,1,3,24,0,0,0,1440.000000,1,0,840.550000,hint,nbiot,60 diff --git a/results/success.png b/results/success.png index 05fca40..376b70a 100644 Binary files a/results/success.png and b/results/success.png differ diff --git a/results/tables-stats.org b/results/tables-stats.org index 2031945..f0368db 100644 --- a/results/tables-stats.org +++ b/results/tables-stats.org @@ -1,33 +1,33 @@ -|simkey |wireless | wakeupfor| sd_success| success| senderEnergy| receiverEnergy| senderUptime| receiverUptime| nSendFail| -|:---------------|:--------|---------:|----------:|-------:|------------:|--------------:|------------:|--------------:|---------:| -|baseline |lora | 60| 0.000000| 0.000| 632.3280| 581.1460| 1440.000| 1439.923| 0| -|extended |lora | 60| 1.505359| 5.735| 982.8520| 613.5576| 2090.090| 1500.527| 0| -|hint |lora | 60| 0.000000| 0.000| 641.9022| 585.4391| 1440.000| 1449.476| 0| -|hintandextended |lora | 60| 1.635282| 6.185| 1027.2315| 620.5401| 2129.752| 1513.171| 0| +|simkey |wireless | wakeupfor| sd_success| success| senderEnergy| receiverEnergy| senderUptime| receiverUptime| sd_success_ccgrid| success_ccgrid| senderEnergy_ccgrid| receiverEnergy_ccgrid| senderUptime_ccgrid| receiverUptime_ccgrid| senderEnergy_err| receiverEnergy_err| senderUptime_err| receiverUptime_err| nSendFail| +|:---------------|:--------|---------:|----------:|-------:|------------:|--------------:|------------:|--------------:|-----------------:|--------------:|-------------------:|---------------------:|-------------------:|---------------------:|----------------:|------------------:|----------------:|------------------:|---------:| +|baseline |lora | 60| 0.000000| 0.000| 632.3280| 581.1460| 1440.000| 1439.923| 0.0| 0.0| 619.73| 579.84| 1440.00| 1440.00| 2| 0| 0| 0| 0| +|extended |lora | 60| 1.505359| 5.735| 982.8520| 613.5576| 2090.090| 1500.527| 2.0| 7.7| 1207.61| 627.35| 2564.53| 1523.52| -23| -2| -23| -2| 0| +|hint |lora | 60| 0.000000| 0.000| 641.9022| 585.4391| 1440.000| 1449.476| 0.0| 0.0| 753.42| 957.64| 1440.00| 2354.94| -17| -64| 0| -62| 0| +|hintandextended |lora | 60| 1.635282| 6.185| 1027.2315| 620.5401| 2129.752| 1513.171| 2.2| 9.8| 1397.98| 663.58| 2904.50| 1602.39| -36| -7| -36| -6| 0| -|simkey |wireless | wakeupfor| sd_success| success| senderEnergy| receiverEnergy| senderUptime| receiverUptime| nSendFail| -|:---------------|:--------|---------:|----------:|-------:|------------:|--------------:|------------:|--------------:|---------:| -|baseline |nbiot | 60| 1.291633| 2.255| 787.9910| 593.6499| 1440.000| 1439.923| 0| -|extended |nbiot | 60| 1.585075| 6.010| 789.8257| 591.7123| 1482.840| 1443.758| 0| -|hint |nbiot | 60| 2.356878| 4.230| 881.7522| 610.6209| 1440.000| 1462.637| 0| -|hintandextended |nbiot | 60| 1.824016| 6.860| 907.3576| 606.1796| 1484.538| 1459.460| 0| +|simkey |wireless | wakeupfor| sd_success| success| senderEnergy| receiverEnergy| senderUptime| receiverUptime| sd_success_ccgrid| success_ccgrid| senderEnergy_ccgrid| receiverEnergy_ccgrid| senderUptime_ccgrid| receiverUptime_ccgrid| senderEnergy_err| receiverEnergy_err| senderUptime_err| receiverUptime_err| nSendFail| +|:---------------|:--------|---------:|----------:|-------:|------------:|--------------:|------------:|--------------:|-----------------:|--------------:|-------------------:|---------------------:|-------------------:|---------------------:|----------------:|------------------:|----------------:|------------------:|---------:| +|baseline |nbiot | 60| 1.291633| 2.255| 787.9910| 593.6499| 1440.000| 1439.923| 1.12| 1.27| 749.69| 591.07| 1440.00| 1440.00| 5| 0| 0| 0| 0| +|extended |nbiot | 60| 1.585075| 6.010| 789.8257| 591.7123| 1482.840| 1443.758| 1.80| 7.16| 946.39| 599.13| 1788.32| 1447.54| -20| -1| -21| 0| 0| +|hint |nbiot | 60| 2.356878| 4.230| 881.7522| 610.6209| 1440.000| 1462.637| 3.45| 9.72| 1007.06| 702.59| 1440.00| 1666.51| -14| -15| 0| -14| 0| +|hintandextended |nbiot | 60| 1.824016| 6.860| 907.3576| 606.1796| 1484.538| 1459.460| 2.36| 9.41| 1066.66| 623.30| 1901.47| 1492.53| -18| -3| -28| -2| 0| -|simkey |wireless | wakeupfor| sd_success| success| senderEnergy| receiverEnergy| senderUptime| receiverUptime| nSendFail| -|:---------------|:--------|---------:|----------:|-------:|------------:|--------------:|------------:|--------------:|---------:| -|baseline |lora | 180| 1.255250| 1.835| 2101.411| 1764.778| 4319.945| 4317.934| 0| -|extended |lora | 180| 1.104752| 9.825| 2168.013| 1765.011| 4761.910| 4355.365| 0| -|hint |lora | 180| 1.522858| 10.050| 2195.374| 2015.582| 4319.945| 4832.081| 0| -|hintandextended |lora | 180| 0.562456| 10.735| 2318.551| 1869.760| 4791.917| 4543.989| 0| +|simkey |wireless | wakeupfor| sd_success| success| senderEnergy| receiverEnergy| senderUptime| receiverUptime| sd_success_ccgrid| success_ccgrid| senderEnergy_ccgrid| receiverEnergy_ccgrid| senderUptime_ccgrid| receiverUptime_ccgrid| senderEnergy_err| receiverEnergy_err| senderUptime_err| receiverUptime_err| nSendFail| +|:---------------|:--------|---------:|----------:|-------:|------------:|--------------:|------------:|--------------:|-----------------:|--------------:|-------------------:|---------------------:|-------------------:|---------------------:|----------------:|------------------:|----------------:|------------------:|---------:| +|baseline |lora | 180| 1.255250| 1.835| 2101.411| 1764.778| 4319.945| 4317.934| 1.40| 1.91| 2026.23| 1755.92| 4320.00| 4320.00| 4| 1| 0| 0| 0| +|extended |lora | 180| 1.104752| 9.825| 2168.013| 1765.011| 4761.910| 4355.365| 0.70| 11.45| 2692.46| 1772.35| 6038.78| 4364.27| -24| 0| -27| 0| 0| +|hint |lora | 180| 1.522858| 10.050| 2195.374| 2015.582| 4319.945| 4832.081| 0.00| 12.00| 2094.09| 2169.50| 4320.00| 5344.86| 5| -8| 0| -11| 0| +|hintandextended |lora | 180| 0.562456| 10.735| 2318.551| 1869.760| 4791.917| 4543.989| 0.24| 11.94| 2757.79| 1867.34| 6155.50| 4600.09| -19| 0| -28| -1| 0| -|simkey |wireless | wakeupfor| sd_success| success| senderEnergy| receiverEnergy| senderUptime| receiverUptime| nSendFail| -|:---------------|:--------|---------:|----------:|-------:|------------:|--------------:|------------:|--------------:|---------:| -|baseline |nbiot | 180| 1.2720418| 9.200| 2582.264| 1766.027| 4319.945| 4317.934| 0| -|extended |nbiot | 180| 1.0582887| 9.975| 2456.997| 1751.549| 4343.803| 4319.913| 0| -|hint |nbiot | 180| 0.6046512| 10.715| 2971.540| 1996.878| 4319.945| 4501.213| 0| -|hintandextended |nbiot | 180| 0.5713204| 10.735| 2949.652| 1975.150| 4334.482| 4482.790| 0| +|simkey |wireless | wakeupfor| sd_success| success| senderEnergy| receiverEnergy| senderUptime| receiverUptime| sd_success_ccgrid| success_ccgrid| senderEnergy_ccgrid| receiverEnergy_ccgrid| senderUptime_ccgrid| receiverUptime_ccgrid| senderEnergy_err| receiverEnergy_err| senderUptime_err| receiverUptime_err| nSendFail| +|:---------------|:--------|---------:|----------:|-------:|------------:|--------------:|------------:|--------------:|-----------------:|--------------:|-------------------:|---------------------:|-------------------:|---------------------:|----------------:|------------------:|----------------:|------------------:|---------:| +|baseline |nbiot | 180| 1.2720418| 9.200| 2582.264| 1766.027| 4319.945| 4317.934| 1.33| 9.90| 2117.65| 1761.78| 4320.00| 4320.00| 18| 0| 0| 0| 0| +|extended |nbiot | 180| 1.0582887| 9.975| 2456.997| 1751.549| 4343.803| 4319.913| 0.87| 11.23| 2311.52| 1760.92| 4868.78| 4323.71| 6| -1| -12| 0| 0| +|hint |nbiot | 180| 0.6046512| 10.715| 2971.540| 1996.878| 4319.945| 4501.213| 0.37| 11.92| 2180.07| 1868.54| 4320.00| 4576.83| 27| 6| 0| -2| 0| +|hintandextended |nbiot | 180| 0.5713204| 10.735| 2949.652| 1975.150| 4334.482| 4482.790| 0.41| 11.88| 2151.14| 1822.44| 4906.10| 4473.32| 27| 8| -13| 0| 0|