#!/usr/bin/env bash tmp=$(mktemp) while IFS= read -r line; do echo $line >> $tmp done first_line=$(head -n1 $tmp|grep "timestamp,nsecs,power") # To check for validity last_line=$(cat $tmp | tail -n1) ncomma=$(echo "$last_line" | tr -cd ',' | wc -c | tr -d ' ') # To check for validity # Check if measurements are valid (not perfect test but sufficient) if [ -z "${first_line}" ] || [ $ncomma -ne 2 ] then cat $tmp exit 100 fi # If valid then proceed to uniqueness cat $tmp | awk -F"," 'BEGIN{LAST=-1;skipped=0} {if(LAST!=$3){print $1","$2","$3;skipped=0}else{skipped=1} LAST=$3} END{if(skipped){print "'$last_line'"}}' rm $tmp