BASH | Shalla | Squid acl import error | blacklist import // deduplicate shalla blacklists

# SLOW

function dedup () {
   declare -a A_IN=("${!1}")
   declare -a A_OUT=("${!1}")

   for A_IN_ELEM in "${A_IN[@]}"; do
    II=-1
    for A_OUT_ELEM in "${A_OUT[@]}"; do
	II=$(( $II + 1 ))
	[[ "x$A_OUT_ELEM" == "x$A_IN_ELEM" ]] && continue
	[[ "x$A_OUT_ELEM" == "x" ]] && continue
	echo "$A_OUT_ELEM" | grep -w "$A_IN_ELEM" > /dev/null
	if [ $? -eq 0 ]; then
	    unset  A_OUT[$II]
	fi
    done
    A_OUT=(${A_OUT[@]})
   done
   for A_OUT_ELEM in "${A_OUT[@]}"; do
    echo > /dev/null
    echo $A_OUT_ELEM | grep -v '^$'
   done
}

OIFS=$IFS;IFS=$'\n'
   A_SRC=($(cat "/PATH/FILE2DEDUP.domains" | grep -v '^#' | grep -v '^\s*$'))
IFS=$OIFS
dedup A_SRC[@]

# FAST

OIFS=$IFS;IFS=$'\n'
A_DOUBLECAND=($(cat "$SH_FILE" | grep -v '^#' | grep -v '^\s*$' | rev | sed 's|\.|\ |g' | awk '{print $1OFS$2}' | sort | uniq -c | grep -v -w '1' | rev | awk '{print $1OFS$2}' | sed 's|\ |\.|g'))
	    IFS=$OIFS

	    GREP_ELEM_STRG=""
	    for C_DEDUP in "${A_DOUBLECAND[@]}"; do
		GREP_ELEM_STRG="$GREP_ELEM_STRG -e '$C_DEDUP'"
	    done
	    
	    [[ "x$GREP_ELEM_STRG" != "x" ]]
	    if [ $? -eq 0 ]; then
		cat "$SH_FILE"  | sort | grep -v '^#' | grep -v '^\s*$' | eval grep -F -v -w $GREP_ELEM_STRG | sed 's|^|\.|'  > "${BLDIR_SQUID}/$SH_TYPE.dstdom.tmp"
		SH_FILE_DIRTY="$(cat "$SH_FILE"  | sort | grep -v '^#' | grep -v '^\s*$' | eval grep -F -w $GREP_ELEM_STRG )"
		LASTLINE="|||"
		SH_FILE_DIRTY_CLEANED="$(echo "$SH_FILE_DIRTY" | awk '{print $1}' | sed 's|\.|\ |g' | rev | sort -u | (
		    while read LINE; do
			echo "$LINE" | grep -w "$LASTLINE" > /dev/null
			if [ $? -ne 0 ]; then
			    echo "$LINE"
			    LASTLINE="$LINE"
			fi
		    done
		) | sed 's|\ |\.|g' | rev | sed 's|^|\.|')"
		echo "$SH_FILE_DIRTY_CLEANED"  >> "${BLDIR_SQUID}/$SH_TYPE.dstdom.tmp"
	    else
		cat "$SH_FILE"  | sort | grep -v '^#' | grep -v '^\s*$' > "${BLDIR_SQUID}/$SH_TYPE.dstdom.tmp"
	    fi   
	    mv -f "${BLDIR_SQUID}/$SH_TYPE.dstdom.tmp" "${BLDIR_SQUID}/$SH_TYPE.dstdom"