#!/bin/bash

###############################################################################
#
# Mirroring Script.
#
# Copyright (c) 2006
# License- 
# Author-  Mike Goins
# Company- Adtec Digital, Inc.
# Date-    2006-08-14
# Modified 2006-10-19
#
# This script is a wrapper around wget and ftp that mirrors content from an  
# ftp server.  The mirrorred files are speficied as a directory or in a list 
# file.  
#
# This script is normally daemonized but also can be run as single batch.
# 
# Current file dependencies:
# /var/local/lib/dpi/scheduler.xml
# /var/local/lib/dpi/channelxx.xml
# /usr/local/lib/mirror/scheduler.xsl
# /etc/sysconfig/mirror.cfg
# /etc/env.global
# /usr/local/bin/rdy
# /usr/local/bin/tnftp
# /usr/local/bin/ftpcopy (newer version than in gnu folders, support >2GB files
##############################################################################
export LD_LIBRARY_PATH=/usr/local/lib

shopt -s -o nounset                       # require all variables to be declared
shopt -s extglob                          # set for matching in variable settings


# bail if root
#if [ $UID -eq 0 ] ;then exit 13; fi  #Perm Denied

# global variables
declare -rx SCRIPT=${0##*/}               # set a variable to contain script name
declare -r SEP="/"                        # Set separator, portability?
declare -ri WAIT_RETRY=3                        # add to cfg? 
declare DEBUG="false"
declare -ri ERR_ARG=2
declare LOCAL_MEDIA_DIR="/media/hd0"              # default, overridden with env.global
declare -ir NUM_DAYS=7                                 # number of days of schedule to get
declare -rx SCH_XML="/var/local/lib/dpi/cfg/scheduler.xml" # sched file. Used to pull HEAD
declare -rx SCH_XSL="/usr/local/lib/mirror/scheduler.xsl" # sched stylesheet. Used to pull HEAD with xsltproc
declare -rx CHAN_XML_DIR="/var/local/lib/dpi/cfg" 
declare -rx CHAN_XML_PREF="chnlmgr_"
declare -rx CHAN_XML_SUF=".xml"
declare -rx CHAN_XSL="/usr/local/lib/mirror/channel.xsl"
declare -rx MIRROR_HELP=/usr/local/bin/mirror_help
declare SCH_DIR="/media/hd0/schins"                # could be overridden in env.global?
declare MVL_DIR="../mvl"
declare RDY_DIR="../rdy"
declare -r MVL_EXT=".MVL"                  
declare -r REJECT_EXT1="*.FEC"           #csv list of extensions NOT to download
declare -r REJECT_EXT2="*.LOG"           #csv list of extensions NOT to download
declare -r REJECT_EXT3="*.log"           #csv list of extensions NOT to download
declare DRIVE_FULL=FALSE
declare -r MIN_FREE_BLOCKS=50000          # ~200 Meg, space minimum required to mirror.
declare -r MAX_FREE_BLOCKS=75000          # ~300 Meg, max space to delete up to.
declare TAG=$'\040\040[MIRROR]'              # Logging tag
declare STARTUP_OK=TRUE                       # flag to see if init conditions OK to run
declare RATELIMIT=3000000                     # init stub for setting rate limited downloads, in Bytes/sec?
declare FTPCOPY="/usr/local/bin/ftpcopy"      # path to updated ftpcopy.  Old version in /bin/ftpcopy
declare SED=/bin/sed
declare SYSLOGCONF=/etc/syslog.conf

# DPI global only
declare REMOTE_MVL_DIR="MirrorListFiles"
declare REMOTE_SCH_DIR="MergedSchedules"
declare REMOTE_RDY_DIR="TransactionLogs"
#declare REMOTE_MED_DIR="MasterVideoLibrary MasterVideoLibrary2"
declare REMOTE_MED_DIR="MasterVideoLibrary2"
declare -i DOWN_OK=0
declare -i DOWN_ERROR=0


# changeble info / in mirror.cfg file
declare HOSTIPADDRESS           
declare HOSTMODE=Client
declare REMOTE_DIR                          # not in API, not exposed
declare CLTUSERPASSWORD=adtec,none
declare FTPCLIENTPASV=passive
declare MIRRORLISTFILE="LIST.MVL"
declare HOSTTIMER=600
declare FTPTIMEOUT=10
declare FTPDATATIMEOUT=45  #not yet implemented
declare DONOTREPLACEMPEG=NO
declare MRVERIFY

# simple list of the above for easy iteration
declare -r SETTINGS_LIST="HOSTIPADDRESS HOSTMODE CLTUSERPASSWORD \
FTPCLIENTPASV MIRRORLISTFILE HOSTTIMER FTPTIMEOUT FTPDATATIMEOUT DONOTREPLACEMPEG MRVERIFY"

if [ -e "/etc/env.global" ]; then
   . "/etc/env.global"
else 
   STARTUP_OK=FALSE
fi

# data validation variables
declare HOSTIPADDRESS_VALID='[a-zA-Z0-9.:]*'
if [ "$PRODUCT" == "dpi" ] ; then 
    declare HOSTMODE_VALID='@(MirrorList|mirrorlist|MIRRORLIST|MirrorClient|Mirrorclient|mirrorclient|MIRRORCLIENT|Client|client|CLIENT|CCMS|Ccms|ccms)'
else
    declare HOSTMODE_VALID='@(MirrorList|mirrorlist|MIRRORLIST|MirrorClient|Mirrorclient|mirrorclient|MIRRORCLIENT|Client|client|CLIENT|MirrorAdtec|mirroradtec|Mirroradtec|MIRRORADTEC)'
fi 

declare REMOTE_DIR_VALID='[a-zA-Z0-9_./]*'
declare CLTUSERPASSWORD_VALID='[a-zA-Z0-9_,]*'
declare FTPCLIENTPASV_VALID='@(passive|active|PASSIVE|ACTIVE|Passive|Active)'
declare MIRRORLISTFILE_VALID='[a-zA-Z0-9_./,]*'
declare HOSTTIMER_VALID='[0-9]*'
declare FTPTIMEOUT_VALID='[0-9]*'
declare FTPDATATIMEOUT_VALID='[0-9]*'
declare DONOTREPLACEMPEG_VALID='@(NO|YES|No|Yes|no|yes)'
declare MRVERIFY_VALID='[0-9]*'

declare CFG_FILE="/etc/sysconfig/${SCRIPT}.cfg"
test -f $CFG_FILE && . $CFG_FILE


# =============================================================================
# =============================================================================
function set_paths () {
   #setup local media dir
   declare TMP
   declare TMP=${CONTENT_PATH%%[:|,]*}   # strip left to either the comma or colona
   if [ "$TMP" ]; then             # if path exists then assign.
      LOCAL_MEDIA_DIR=$TMP
   fi
   if [ ! -e $LOCAL_MEDIA_DIR ]; then
      STARTUP_OK=FALSE
   fi
   if [ "$PRODUCT" == "dpi" ]; then
      if [ ! -d "$SCH_DIR" ]; then
         STARTUP_OK=FALSE
      fi
      if [ ! -d "$LOCAL_MEDIA_DIR$SEP$RDY_DIR" ]; then
         STARTUP_OK=FALSE
      fi
   fi
   if [ ! -e $LOCAL_MEDIA_DIR$SEP$MVL_DIR ]; then
      STARTUP_OK=FALSE  
   fi
   if [ ! -e "${LOCAL_MEDIA_DIR}${SEP}${RDY_DIR}${SEP}.excess" ]; then
      touch "${LOCAL_MEDIA_DIR}${SEP}${RDY_DIR}${SEP}.excess"
   fi   
}


function show_main_help (){
   declare var
   cat <<EOF
$SCRIPT - client file transfer and synchronization system.  This is designed to
add files to the local filesystem either by processing a file list or by
mirroring files of an FTP server.
List of available settings:
EOF
for var in $1; do printf "%s\n" $var; done
cat <<EOF
Help for each setting can be accessed with MIRRORHELP <SETTING>
EOF
}


# =============================================================================
# =============================================================================
function show_config (){
   # $1: normally list of settings
   local var
   for var in $1;
   do
       echo -n ${var}"="
       eval echo \$$var
   done

}

function update_syslog()
{
    $SED -i "s/\(local1.notice.*@\)\(.*\)/\1${1}/" $SYSLOGCONF
    pkill -HUP syslogd
}

# =============================================================================
# =============================================================================
function settings_change () {
    # $1: setting to be changed, $2: setting new value, $3: config file
    # The function find a match for the setting to be updated, validates
    # the new value and call the function to update the actual value. 
    # if no config file ( $# = 2 ) then it dynamically sets the variable (late bind)
    local VAR
    local RET=0
    local ESC_2
    #printf "%s\n" "${1}=${2}" 
            
    eval VAR=\$${1}_VALID &>/dev/null    #raise the data validation setting for that variable.
    if  [ "$VAR" ] && [[ $2 = $VAR ]]; then             # test if valid
        if [ $# -eq 3 ]; then            # to update file
            ESC_2=${2//\//\\/}   # escape the slashes
                #sed -i "s/\(^${1}=\)\(\"\)*.*\(\"\)/\1\2${2}\3/" $3
                vi -e -n -c "%s/\(^${1}=\)\(\"\)*[^\"]*\(\"\)*/\1\2${ESC_2}\3/ | w! | q" $3 &>/dev/null
            RET=$?
            if [ $RET -eq 0 -a "$1" == "HOSTMODE" ]; then  # restart mirroring
               /etc/init.d/mirrord restart &>/dev/null
            fi
            if [ "$1" == "HOSTIPADDRESS" ]; then  # update syslog
               update_syslog "${2}"
            fi
        else
            eval "$1"="$2"               # dynamic assignment only
            RET=$?
        fi
    else
        RET=$ERR_ARG
    fi
    return $RET
}


# =============================================================================
# Function GET_SCHEDULE_FOR
# Retrieves required schedules by calculating the schedulename for the next 
#  NUM_DAYS. 
# Args:
#    $1  HeadChan from XSLT proc
# =============================================================================
function get_schedule_for () {
    local SCHNAME
    for (( CNT=0; CNT<$NUM_DAYS; CNT++ )); do
       SCHNAME="$(printf "%X\n" $(date -d "${CNT} day" +"%-m"))$(date -d "${CNT} day" +"%d")${1}.SCH" 
       ftp_cmd "${SCH_DIR}" "${REMOTE_SCH_DIR}${SEP}${SCHNAME}" "NO"  # NO overrides DONOTREPALCEMPEG
       #echo "${SCH_DIR}" "${REMOTE_SCH_DIR}${SEP}${SCHNAME}"
       #wget_cmd "${LOCAL_MEDIA_DIR}${SEP}$SCH_DIR" "${REMOTE_SCH_DIR}${SEP}${SCHNAME}"
    done
}

# =============================================================================
# function GET_LIST
# Gets the remote file list
# =============================================================================
function get_list () {

            FINALLISTFILE=/media/hd0/mvl/.Ffiles.txt
            local TEMPFILE1=/media/hd0/mvl/.filelisting.txt
            local TEMPFILE2=/media/hd0/mvl/.files.txt
            
            depth=10       #number of sub directory levels to go into
         
            #grab all directory and filenames of the depth specified
            ftpls -m --max-depth=$depth -a -u ${CLTUSERPASSWORD%,*} -p ${CLTUSERPASSWORD#*,} ftp://${HOSTIPADDRESS}/ > $TEMPFILE1 2>/dev/null
            
            #when outputed each dir/file has 4 lines of data, this counts the total number lines                        
            local NUMFILES=`wc -l < $TEMPFILE1`


            #remove size and timestamp from listing, keeping name and type
            #listing for each file/dir goes Name/Type/Time/Size            
            for (( i=1; i<=NUMFILES; i=$(($i+4)) ))
            do
            N=$i           
            K=$(($i+1))
            sed -n "${N}p;${K}p;${K}q" $TEMPFILE1                
            done > $TEMPFILE2
            
            #make sure the file is empty before apending
            rm $FINALLISTFILE 2>/dev/null
            touch $FINALLISTFILE
            
            #read through the file list and remove all directories and repeats of files                                               
            while read line;
            do
            
            #if this line is the files name store it
            if [ "`expr substr $line 1 1`" == "N" ]; then           
               PASTLINE=$line
            
            #if the line is the type and the type a file, check if its a repeat and if not the store it to Final file list
            elif [ "$line" == "Kfile" ]; then
               
               
               if [ "`expr substr ${PASTLINE##N} 1 1`" != "/" ]; then                                                     
                  #echo ${PASTLINE##N} 
                  echo ${PASTLINE##N} >> $FINALLISTFILE
               fi
            fi            
            done < $TEMPFILE2

}

# =============================================================================
# function DEL_SCHEDULE_FOR
# Removes old scehdules by date
# =============================================================================
function del_schedule_for () {
    # $1 HEADCHAN
    # calculate the schedulename for the prev NUM_DAYS and delete them.
    local SCHNAME
    #local CNT=-1;
    for (( CNT=-$NUM_DAYS; CNT<0; CNT++ )); do
        SCHNAME="$(printf "%X\n" $(date -d "${CNT} day" +"%-m"))$(date -d "${CNT} day" +"%d")${1}.SCH" 
        if [ -e ${SCH_DIR}${SEP}${SCHNAME} ]; then
           rm -f ${SCH_DIR}${SEP}${SCHNAME}
           logger -p user.info -t "$TAG" "Deleted: ${SCH_DIR}${SEP}${SCHNAME}"
        fi
    done
}

function ftp_cmd () {
    # $1 is the local dir to dowloand, $2 is the file
    # if there is a $3, then it is to override the DONOTREPLACE for MVL/SCH

    if [[ ${HOSTIPADDRESS} != *:* ]]; then
        HOSTIPADDRESS="ftp://${HOSTIPADDRESS}:21"
    fi
        
    # first get the remote dir
    # this is because ftpcopy does not handle remote filename globbing.  This allows us to do it ourselves
    # locally
    
    # test if file ends in asterisk, then we need to download remote list
    if [ "${2:(-1)}" == "*" ]; then
      
       # get the parent dir for the remote file ( /foo/bar* -> /foo)
       declare REM_BASE="${2%/*}"   # this gets the parent, but if no slashes gets the entire      
       if [[ ${REM_BASE} != */* ]]; then   # no slashes set as /
          REM_BASE="/"
       fi
       echo $REM_BASE
       /usr/bin/ftpls -u ${CLTUSERPASSWORD%,*} -p ${CLTUSERPASSWORD#*,} ftp://${HOSTIPADDRESS} $REM_BASE > ${1}/.list  # gets the remote folder list into a local file
       declare REM_FILE="${2##*/}"      # basename
       declare REM_MATCH=$(grep -h -o -m 1 ${1}/.list)    # grep the match             
       
       # if no match then should fail out here
       if  [ -z "REM_MATCH" ]; then
          logger -p user.info -t "$TAG"
          return 1
       fi
       declare REM_PATH="${REM_BASE}/${REM_MATCH}"
    else
       declare REM_PATH="${2}"      
    fi   
    declare CMD="--rate-limit=${RATELIMIT} --timeout=${FTPTIMEOUT} --exclude=${REJECT_EXT1} --exclude=${REJECT_EXT2} --exclude=${REJECT_EXT3} --no-delete "
    #if [ "$FTPCLIENTPASV" != "passive" ]; then
    #    CMD=${CMD}"-A "
    #fi
            
    declare LOCAL_PATH_EXT=$(dirname $2)   #Directory extension of the file to downlaod
    LOCAL_PATH=`echo ${1%%/}`  #removes '/' at the end of local dir pathname                  
   
    declare LOCAL_BASE=$(basename $2)   #Filename with extension 
    declare LOCAL_EXT=${LOCAL_BASE##*.} #Only extension of the file
    LOCAL_BASE=$(basename $LOCAL_BASE "."$LOCAL_EXT)  #Strips extension off of the filename    
    
    #Set isMPG false to start, check if the file and extension are in the freg database
    declare isMPG=FALSE  
    
    declare numLines=`/usr/local/bin/sqlite3 /media/hd0/.meta/freg.db "select count(*) from tb_file where tb_file.muxtype != 0 and tb_file.path == \"${LOCAL_PATH}\" and tb_file.basename == \"$LOCAL_BASE\" and tb_file.extension == \"$LOCAL_EXT\""`   
   
        

    if [ $numLines  -gt 0 ]; then #if the file shows up in the database then it is an MPG      
         isMPG=TRUE                 
    fi
    

    if [ "`echo $DONOTREPLACEMPEG |tr '[:lower:]' '[:upper:]'`" == "YES" ] && [ "$isMPG" == "TRUE" ]; then               
        declare CMD="${CMD} --ignore-time --ignore-size"
    fi
    CMD=" -u ${CLTUSERPASSWORD%,*} -p ${CLTUSERPASSWORD#*,} ${CMD} ${HOSTIPADDRESS}${SEP}${REM_PATH} ${1}"
    
    #echo "${REM_MATCH}"
    #CMD=" -u ${CLTUSERPASSWORD%,*} -p ${CLTUSERPASSWORD#*,} ${CMD} ${HOSTIPADDRESS}${REM_PATH} ${1}"    
    
    if [ "$DEBUG" = "true" ]; then
        #echo "$FTPCOPY --bps --progress --loglevel=3 ${CMD}" 
        $FTPCOPY --bps --progress --loglevel=3 ${CMD}                
    else
        #logger -p user.info -t "${TAG}" "$FTPCOPY --loglevel=0 ${CMD}"        
        $FTPCOPY --loglevel=0 ${CMD} # &> logger -p user.info -t \"${TAG}\" 
    fi
}



# =============================================================================
# function WGET_CMD
# Handles determination if file needs to be download. 
# Args:
#    $1 is the local dir to dowloand to, $2 is the file
#
# TODO:   Scrape the 'recursive' setting for the media_paths in env.global 
# and as as arguments here : "1-r l inf"
# =============================================================================
function wget_cmd () {
    local RET  # holder for return value
    local EXISTS
    if [ ! -e ${1}${SEP}${2##*/} ] ; then # file does not exist
       EXISTS="FALSE"
    fi
    # $1 is the local dir to dowloand, $2 is the file, $3 is DONOTREPLACEMPEG override
    # $3 is just still a test.  We all going to set this flag when we know we are mirroring non-media files. Eventually this
    # will need to tap into freg database.
   
    # TODO:   Scrape the 'recursive' setting for the media_paths in env.global and as as arguements here : "1-r l inf"
    if [ "`echo $DONOTREPLACEMPEG |tr '[:lower:]' '[:upper:]'`" == "YES" ] && [ $# -eq 2 ]; then 
        declare CMD="wget -r -nc -l inf --directory-prefix=${1} --no-host-directories"
    else
        declare CMD="wget --mirror --directory-prefix=${1} --no-host-directories"
    fi
    CMD="${CMD} --reject=${REJECT_LIST} --no-directories --timeout=${FTPTIMEOUT} --waitretry=${WAIT_RETRY} "
    if [ "$FTPCLIENTPASV" = "passive" ]; then
        CMD=${CMD}"--passive-ftp "
    fi
    if [ "$DEBUG" = "false" ]; then
        CMD=${CMD}"--quiet ftp://${CLTUSERPASSWORD/,/:}@${HOSTIPADDRESS}${SEP}${2}"
        $CMD  
        RET=$?
        if [ $RET -ne 0 ] && [ ! -e ${1}${SEP}${2##*/} ]; then
           logger -p user.error -t "$TAG" "ERROR: ${RET}: ${HOSTIPADDRESS}${SEP}${2}"
           let "DOWN_ERROR++"
    else
           if [ "$EXISTS" ==  "FALSE" ]; then 
              logger -p user.info -t "$TAG" "Initial Download OK: ${HOSTIPADDRESS}${SEP}${2}"
              let "DOWN_OK++"
           fi
        fi       
    else
        CMD=${CMD}" ftp://${CLTUSERPASSWORD/,/:}@${HOSTIPADDRESS}${SEP}${2}"
    $CMD
    fi   
}


function mirror_list () {
   # it is possible to have CSV for mirrorlist variable.  So we will iterate over each one, 
   # retrieve it, process it.
       
   local PATH_LIST=/media/hd0/mvl   ####  HARD CODED  #####  CHANGE  IT ####################
   local OLD_IFS="${IFS}"
   
   for FILE in $MIRRORLISTFILE; do             
          IFS="${OLD_IFS}"          
          ftp_cmd $PATH_LIST $FILE "NO"
          if [ -e "$PATH_LIST$SEP$FILE" ]; then
             process_list $PATH_LIST$SEP$FILE        # process the file
          else
             logger -p user.error -t "$TAG" "Failed to get: $FILE"
          fi
          IFS=","                     
 done
  IFS="${OLD_IFS}"              # restore IFS
}


# =============================================================================
# =============================================================================
function ccms () {
    # find the heads in the schedule XML file.  Get then, process then.
    # the following grep is very sensitve to the XML formatting.  It is expected that the 
    # HEAD is exactly three characters and wrapped by <headend_id> and </headend_id>:
    # e.g., <headend_id>888</headend_id>
    declare -a HEADCHAN=(`xsltproc $SCH_XSL $SCH_XML`)  # put it in an array
    declare -i CHAN_CNT=0  # count to iterate over HEADCHAN
    for i in ${HEADCHAN[*]}; do
        del_schedule_for $i   # purge old schedules  
        declare CHAN=$(printf "%02d" ${CHAN_CNT})
        declare XML_FILE="${CHAN_XML_DIR}${SEP}${CHAN_XML_PREF}${CHAN}${CHAN_XML_SUF}"
        declare CHAN_STATUS=`xsltproc "$CHAN_XSL" "$XML_FILE" | cut -d"," -f3` 
        if [ "$CHAN_STATUS" == "ON" ]; then
           get_schedule_for $i
           local LIST_NAME="${i/#??}${MVL_EXT}"  # this does not remove duplicate HEADS.  
           #  HEAD=${HEADCHAN[@]/#??/}  Future use?
           ftp_cmd "$LOCAL_MEDIA_DIR$SEP$SEP$MVL_DIR" "$REMOTE_MVL_DIR$SEP$LIST_NAME" "NO"
           if [ -e "$LOCAL_MEDIA_DIR$SEP$MVL_DIR$SEP$LIST_NAME" ]; then
                   #echo "$LOCAL_MEDIA_DIR$SEP$MVL_DIR$SEP$LIST_NAME"
              process_list "$LOCAL_MEDIA_DIR$SEP$MVL_DIR$SEP$LIST_NAME"   
              rdy "F" $LIST_NAME  
           else
              logger -p user.error -t "$TAG" "Failed to get: $REMOTE_MVL_DIR$SEP$LIST_NAME"
           fi
    fi    
    let "CHAN_CNT=CHAN_CNT+1"
    done
}    


# =============================================================================
# =============================================================================
function test_drive_space ()
{
   #echo "tds: $1"
   #  Use blocks to avoid complications with M and B human displays.
   let "BL=$(stat -f ${LOCAL_MEDIA_DIR} -c '%f')"  # get free blocks
   #echo "$BL ${1}" 
   if [ ${BL} -lt ${1} ] ; then 
      DRIVE_FULL="TRUE" 
   else
      DRIVE_FULL="FALSE"   # need to clear in case set previously and files deleted. 
   fi   
   #echo "DF: $DRIVE_FULL"
}


# =============================================================================
# =============================================================================
#  delete files from the drive if full, not yet implemented
function clean_drive ()
{
   declare OLDEST_FILE                      # holds the oldest file to delete if needed.
   if [ "$PRODUCT" = "dpi" ]; then
       declare -i CNT=0
       declare EXCESS_FILE="${LOCAL_MEDIA_DIR}${SEP}${RDY_DIR}${SEP}.excess"
       declare -i MAX_CNT=`wc -w ${EXCESS_FILE} | cut -d" " -f1`    # number of entries in excess file.
       declare EXCESS_DATA=$(cat "${EXCESS_FILE}")
       declare -a EXCESS=($EXCESS_DATA)   # excess now in array
       while [ $CNT -lt $MAX_CNT ] && [ $DRIVE_FULL = "TRUE" ]; do
           # we loop through the excess array and delete
           rm -f ${EXCESS[$CNT]}  # umm, do I need path here?
           logger -p user.info -t "$TAG" "Deleted ${EXCESS[$CNT]} to free up drive space"
           test_drive_space ${1}  # here we test if we have deleted up to the MAX needed
           let CNT=$CNT+1
       done
   else
     case "$HOSTMODE" in 
	    MirrorList|mirrorlist|Mirrorlist|MIRRORLIST)
   	       DRIVE_FULL="TRUE"
	       NOT_IN_LIST=$(ls -1 ${LOCAL_MEDIA_DIR} | join -v 1 - "$LOCAL_MEDIA_DIR$SEP$MVL_DIR$SEP$MIRRORLISTFILE" | head -1)
		   #echo "NIL: $NOT_IN_LIST"
           while [ -n "$NOT_IN_LIST" ] && [ $DRIVE_FULL = "TRUE" ]; do
		      logger -p user.info -t "$TAG" "Deleting '${NOT_IN_LIST}' to free up drive space"
		      find ${LOCAL_MEDIA_DIR} -name "$NOT_IN_LIST" -exec rm {} \;
              #find ${LOCAL_MEDIA_DIR} -name "$NOT_IN_LIST" -print
              test_drive_space ${1}   # here we test if we have deleted up to the MAX needed
	          NOT_IN_LIST=$(ls -1 ${LOCAL_MEDIA_DIR} | join -v 1 - "$LOCAL_MEDIA_DIR$SEP$MVL_DIR$SEP$MIRRORLISTFILE" | head -1)			
 		   done
		   ;;
		*)  
          local DATE_RANGE="365 180 90 30 14 7 1"            # range of days to check
   	      #DRIVE_FULL="TRUE"
          for DT in $DATE_RANGE; do 
             OLDEST_FILE=`find ${LOCAL_MEDIA_DIR} -type f -daystart -not -mtime -${DT} -print | head -1`  # get one old file
		     #echo "old: $OLDEST_FILE"
             while [ -n "$OLDEST_FILE" ] && [ $DRIVE_FULL = "TRUE" ]; do
                #echo "rm -f $OLDEST_FILE"
                rm -f "$OLDEST_FILE"
                logger -p user.info -t "$TAG" "Deleted '${OLDEST_FILE}' to free up drive space"
                test_drive_space ${1}   # here we test if we have deleted up to the MAX needed
                OLDEST_FILE=`find ${LOCAL_MEDIA_DIR} -type f -daystart -not -mtime -${DT} -print | head -1`  # get one old file
             done    
          done
	      ;;
	   esac
   fi    
}

# =============================================================================
# Function to transfer the current log back to the argued server
# =============================================================================
function rverify () {
if [ ${MRVERIFY} -gt 0 ]; then
   /usr/local/bin/transfer ${1} put /var/log/all all.log
   for a in `seq 2 ${MRVERIFY}`; do
      let "a=$a-1"
      month=`date --date "${a} day ago" +%m`
      day=`date --date "${a} day ago" +%d`
      year=`date --date "${a} day ago" +%Y`
      /usr/local/bin/transfer ${1} put /var/log/all all-${year}${month}${day}
   done
fi



   for a in `seq ${MRVERIFY}`; do
       month=`date --date "${a} day ago" +%m`
       day=`date --date "${a} day ago" +%d`
       year=`date --date "${a} day ago" +%Y`
       #hex_month=`printf "%X" ${month}`
    
       echo "all-${year}${month}${day}"
done


    /usr/local/bin/transfer ${1} put /var/log/all all.log
}


# =============================================================================
# =============================================================================
function main_proc () {
    DOWN_OK=0
    DOWN_ERROR=0
    if [ "$STARTUP_OK" != "FALSE" ]; then
        case "$HOSTMODE" in
    	    MirrorClient|mirrorclient|Mirrorclient|MIRRORCLIENT)
                logger -p user.info -t "$TAG" "Started ${HOSTMODE} process with ${HOSTIPADDRESS}."
                test_drive_space $MIN_FREE_BLOCKS
                if [ $DRIVE_FULL = "TRUE" ]; then 
                    clean_drive $MAX_FREE_BLOCKS 
                fi
            
            if [ "$DEBUG" = "true" ]; then
            echo Getting remote file list
            fi
            get_list  #get remote file list
            if [ "$DEBUG" = "true" ]; then
            echo Remote file list obtained
            fi

                        
            OLDMIRRORLISTFILE=$MIRRORLISTFILE
            MIRRORLISTFILE=.Ffiles.txt
            logger -p user.info -t "$TAG" "Started ${HOSTMODE} process with ${HOSTIPADDRESS}."
                test_drive_space $MIN_FREE_BLOCKS
                if [ $DRIVE_FULL = "TRUE" ]; then 
                    clean_drive $MAX_FREE_BLOCKS
                fi
                mirror_list
                #rverify "HIP_FORCE" # MPG: deprecated
                logger -p user.info -t "$TAG" "Checked ${DOWN_OK} files, ${DOWN_ERROR} failed."
            
            MIRRORLISTFILE=$OLDMIRRORLISTFILE
            
            
   
			   logger -p user.info -t "$TAG" "Mirror completed."
            ;;
    	    MirrorAdtec|mirroradtec|Mirroradtec|MIRRORADTEC)
                logger -p user.info -t "$TAG" "Started ${HOSTMODE} process with ${HOSTIPADDRESS}."
                test_drive_space $MIN_FREE_BLOCKS
                if [ $DRIVE_FULL = "TRUE" ]; then 
                    clean_drive  
                fi
				$FTPCOPY -u ${CLTUSERPASSWORD%,*} -p ${CLTUSERPASSWORD#*,} --dry-run ${HOSTIPADDRESS} /hd0/media/aquota.user . > /dev/null 2>&1
				if [ $? -eq 0 ]; then
                    ftp_cmd "${LOCAL_MEDIA_DIR}" "${REMOTE_DIR}hd0/media/" 
              		logger -p user.info -t "$TAG" "Mirror completed."
				else
              		logger -p user.info -t "$TAG" "Remote folders hd0/media not found.  Aborting."
				fi
            ;;
            MirrorList|mirrorlist|Mirrorlist|MIRRORLIST)
                logger -p user.info -t "$TAG" "Started ${HOSTMODE} process with ${HOSTIPADDRESS}."
                test_drive_space $MIN_FREE_BLOCKS
                if [ $DRIVE_FULL = "TRUE" ]; then 
                    clean_drive $MAX_FREE_BLOCKS
                fi
                mirror_list
                #rverify "HIP_FORCE" # MPG: deprecated
                logger -p user.info -t "$TAG" "Checked ${DOWN_OK} files, ${DOWN_ERROR} failed."
            ;;
            CCMS|Ccms|ccms)
                if [ $PRODUCT = "dpi" ]; then
                   logger -p user.info -t "$TAG" "Started ${HOSTMODE} process with ${HOSTIPADDRESS}."
                    test_drive_space $MIN_FREE_BLOCKS
                    if [ $DRIVE_FULL = "TRUE" ]; then 
                        clean_drive $MAX_FREE_BLOCKS
                    fi
                   ccms
                    logger -p user.info -t "$TAG" "Checked ${DOWN_OK} files, ${DOWN_ERROR} failed."
                else
                   logger -p user.warn -t "$TAG" "Invalid HOSTMODE: ${HOSTMODE} for ${PRODUCT}"
                fi 
                ;;
            Client|client|CLIENT)
                # do nothing, mirroring is off
            ;;
            *)
               # invalid setting
                logger -p user.warn -t "$TAG" "Bad MirrorMode"
            ;;
        esac
    else
      logger -p user.warn -t "$TAG" "Mirroring did not find required media directories to operate."
    fi    
}


# =============================================================================
# =============================================================================
function process_list () {
    # $1 is a mirror list file
    # This function processes a single list file and it's directives.  It also stores data in 
    # the RDY dir to be used later by the RDY command.
    local NAMED="true"  #this assumes bare filename and toggle to off if name directive not match
    local NAME_ON="#UNITS_NAMED"
    local NAME_OFF="#UNITS_NAMED_END"
    local DELETED=false  # flag to handle delete directive
    local DEL_ON="#MVL_DELETE_FILES_ON"
    local DEL_OFF="#MVL_DELETE_FILES_OFF"
    local MATCH_STATUS="false"
    local V_CHG_ON="#PARAM_CHANGE_ON"
    local V_CHG_OFF="#PARAM_CHANGE_OFF"
    local V_CHG_RESTORE="#PARAM_CHANGE_RESTORE"
    local U_SECT=false                                #flag for settings change
    local CLEAN_STR
    local MVL_BASE=`basename $1`
    #dpi!!!
    local MIR_VER1_ON="#MVL_MIRROR_FILES_VER1_ON"
    local MIR_VER1_OFF="#MVL_MIRROR_FILES_VER1_OFF"
    local MIR_ON="#MVL_MIRROR_FILES_ON"
    local MIR_OFF="#MVL_MIRROR_FILES_OFF"  
    local VER1_STATUS=false       
    local MIR_STATUS=false
    local TMPNAME            #localize processing var
    local MVL_FILES_TO_GET="${LOCAL_MEDIA_DIR}${SEP}$RDY_DIR${SEP}.mvl_get.${MVL_BASE}"
    local MVL_FILES_MISSING="${LOCAL_MEDIA_DIR}${SEP}$RDY_DIR${SEP}.mvl_missing.${MVL_BASE}"
    local MVL_FILES_TO_DEL="${LOCAL_MEDIA_DIR}${SEP}$RDY_DIR${SEP}.mvl_del.${MVL_BASE}"
    local MVL_FILES_GOT="${LOCAL_MEDIA_DIR}${SEP}$RDY_DIR${SEP}.mvl_got.${MVL_BASE}"
    local DIR
    local -i MM
    local FILE_TYPE
    local FREGD=/usr/local/sbin/fregd

    # clean the dir
    >$MVL_FILES_TO_GET
    >$MVL_FILES_TO_DEL
    >$MVL_FILES_MISSING
    >$MVL_FILES_GOT
    
    #cat $LOCAL_MEDIA_DIR$SEP$1
    cat $1 | while read LINE ; do
        #echo $LINE
        LINE=$(echo $LINE | tr -d '\015')   # delete the newlines from winblows text files
        case "$LINE" in
           "//"*|"")
               # echo "Comment Found $LINE"
               #ignore comments and blank lines
               ;;
            [#]*)  # directive
                #echo "Directive Found $LINE"
                # need to change this to CASE statement?
                if [[ "$LINE" = \#UNITS_NAMED\ * ]]; then
                    #echo "found name on"
                    for NAME in ${LINE#${NAME_ON}*}; do              # iterate across space separated variables
                        MM=`expr $HOSTNAME : "$NAME"`
                        if [ $MM -ne 0 ] ; then     # then test for a match.
                           MATCH_STATUS="true"
                        fi
                        #echo "NAMED false"
                    done
                    if [ $MATCH_STATUS = "false" ]; then
                       NAMED="false"
                    fi
                    MATCH_STATUS="false"   # reset match status 
                elif [ "$LINE" = "$NAME_OFF" ] ; then 
                    NAMED="true"

                # handle the DELETE directive.
                elif [ "$LINE" = "$DEL_ON" ]; then
                    DELETED="true"
                elif [ "$LINE" = "$DEL_OFF" ]; then
                    DELETED="false"

                # following section handles dynamic setting of variables.
                elif [ "$LINE" = "$V_CHG_ON" ]; then
                    #echo Start Param Change section
                    U_SECT=true
                elif [ "$LINE" = "$V_CHG_OFF" ] ; then
                    #echo Stop Param Change Section
                    U_SECT=false
                elif [ "$LINE" = "$V_CHG_RESTORE" ] ; then
                    #echo Restore Param
                    test -f $CFG_FILE && . $CFG_FILE

                # dpi handler
                #elif [[ $LINE == *$MIR_VER1_ON ]]; then
                elif [[ $LINE == *MVL_MIRROR_FILES_VER1_ON* ]]; then
                    VER1_STATUS=true
                    #echo "VER1_STATUS is true"
                elif [[ $LINE == *MVL_MIRROR_FILES_VER1_OFF* ]]; then
                    VER1_STATUS=false
                    #echo "VER1_STATUS is false"
                elif [[ $LINE == *MVL_MIRROR_FILES_ON* ]]; then
                    MIR_STATUS=true
                elif [[ $LINE = *MVL_MIRROR_FILES_OFF* ]]; then
                    MIR_STATUS=false
                else
                     : echo "Unmatched directive: $LINE"
                     
                fi
                ;;

            *)
                # must be a bare word
                #echo "Bare: $LINE"
                if [ "$NAMED" == "true" ] ; then
                    if [ "$DELETED" == "false" ]; then
                        if [ "$U_SECT" == "true" ]; then  # && [ -n $FILENAME ]; then
                           # must be a parameter to be set
                            CLEAN_STR=${LINE%;*}   #remove anything up to a ';' to remove potential malicious code
                            settings_change "${CLEAN_STR%=*}" "${CLEAN_STR#*=}"
                        else               
                             
                            if [ "$VER1_STATUS" = "true" ]; then   # dpi ver1, cut and process
                                TMPNAME=`echo ${LINE} | cut -d"\"" -f2`   #chop out name, remove quotes and newline \015
                                echo  $TMPNAME>> $MVL_FILES_TO_GET 
                                for DIR in $REMOTE_MED_DIR; do
                                    ftp_cmd "$LOCAL_MEDIA_DIR" "$DIR$SEP$TMPNAME"
                                    if [ $? -eq 0 ]; then break; fi   # file was found so break out of loop
                                done

                                #wget_cmd $LOCAL_MEDIA_DIR $REMOTE_MED_DIR1$SEP$TMPNAME  #not any way to avoid checking both?
                                #if [ $? -ne 0 ]; then
                                #    wget_cmd $LOCAL_MEDIA_DIR $REMOTE_MED_DIR2$SEP$TMPNAME
                                #fi
                                local FND=`find "$LOCAL_MEDIA_DIR" -wholename "$TMPNAME" -print | wc -l`
                                #if [ `find ${LOCAL_MEDIA_DIR}/* -name "$TMPNAME" -print | wc -l` -eq 0 ] ; then 
                                if [ $FND -eq 0 ] ; then 
                                    echo -e "${MVL_BASE}:$LINE\015">> $MVL_FILES_MISSING  
                                else
                                    #FILE_TYPE=`$FREGD $LOCAL_MEDIA_DIR$SEP$TMPNAME | grep -m 1 MUXTYPE | cut -d= -f2`
                                    #echo -e "${TMPNAME}: ${FILE_TYPE}\015" >> $MVL_FILES_GOT
                                    echo -e "${TMPNAME}\015" >> $MVL_FILES_GOT
                                fi

                            elif [ "$MIR_STATUS" = "true" ]; then   # dpi, cut and process
                                TMPNAME=`echo -e $LINE | cut -d"\"" -f2`"*"   #chop out name and quotes, append asterisk without expanding
                                #if [ ${#TMPNAME} -eq 9 ]; then
                                #        TMPNAME="000"${TMPNAME}  
                                #    fi
                                #echo  ${TMPNAME/\\*/} >> $MVL_FILES_TO_GET 
                                echo  ${TMPNAME} >> $MVL_FILES_TO_GET 
                
                                for DIR in $REMOTE_MED_DIR; do
                                    ftp_cmd "$LOCAL_MEDIA_DIR" "$DIR$SEP$TMPNAME" 
                                    if [ $? -eq 0 ]; then break; fi   # file was found so break out of loop
                                done
                                local FND=`find ${LOCAL_MEDIA_DIR}/* -wholename "${TMPNAME}" -print | wc -l`
                                #echo $FND
                                if [ $FND -eq 0 ] ; then 
                                    echo -e "${MVL_BASE}:${LINE//\"/}\015" >> $MVL_FILES_MISSING  
                                else
                                    #FILE_TYPE=`$FREGD $LOCAL_MEDIA_DIR$SEP$TMPNAME | grep -m 1 MUXTYPE | cut -d= -f2`
                                    echo -e "${TMPNAME}: ${FILE_TYPE}\015" >> $MVL_FILES_GOT
                                    #echo -e "${TMPNAME}\015" >> $MVL_FILES_GOT
                                fi
                            else
                                #if [ "$VER1_STATUS" = "false" ] && [ "$MIR_STATUS" = "false" ] ; then  # non-DPI
                                if [ -n "$REMOTE_DIR" ]; then
                                    ftp_cmd "$LOCAL_MEDIA_DIR" "$REMOTE_DIR$SEP$LINE" 
                                else
                                    ftp_cmd "$LOCAL_MEDIA_DIR" "$LINE" 
                                fi 
                            fi
                        fi
                    else    # deleted is true
                        #elif [ $DELETED="true" ]; then
                        echo  $LINE >> $MVL_FILES_TO_DEL
                        rm -f $LOCAL_MEDIA_DIR$SEP$LINE
                        logger -p user.info -t "$TAG" "Deleted: $LOCAL_MEDIA_DIR$SEP$LINE"
                    fi
                fi
                ;;
            esac
    done
   
  
   } 





# =============================================================================
# Main program start
# =============================================================================
# import and parse MEDIAPATH
# probably need to tuck this call in a better place.
set_paths

if [ $# -eq "1" ]; then
    case "$1" in
        "CONFIGURATION"|"CONFIG"|"CFG")       
            show_config "$SETTINGS_LIST"
            ;;
        "HELP")  
            show_main_help "$SETTINGS_LIST"
            ;;
        "MIRRORHELP")  
            show_main_help "$SETTINGS_LIST"
            ;;
        "DAEMON")
                   # check if IP set, if not, exit.
                   if [ -z $HOSTIPADDRESS ]; then  # host not set
                        exit 5
                   fi
                   # started in daemon mode
                   logger -p user.debug -t "$TAG" "Started in daemon mode"
                   while true
                   do
                       test -f $CFG_FILE && . $CFG_FILE
                       main_proc
                       sleep $HOSTTIMER
                   done
                   logger -p user.debug -t "$TAG" "Ended daemon mode"
                   exit 0;;
        "DEBUG")
                   DEBUG="true"
                   main_proc
                   exit 0;;

        *)         eval SETTING=\$$1 &>/dev/null
                   echo ${SETTING:-undefined};;
    esac
    exit 0

elif [ $# -eq "2" ]; then  # settings update or help
    if [ "$1" == "HELP" -o "$1" == "MIRRORHELP" ]; then
        test -f $MIRROR_HELP && . $MIRROR_HELP && settings_help $2      
        #settings_help $2
    else
        #if [ $UID -ne 0 ]; then 
	#    exit 2   # must be su 
	#fi
	#printf "%s\n" "${1}=${2}" 
            
        settings_change $1 $2 $CFG_FILE   #update the setting file
    fi
    exit $?
elif [ $# -gt "2" ]; then
    exit 2 
else
    # must be Zero args!

    if [ -z $HOSTIPADDRESS ]; then  # host not set
        exit 5
    fi
    logger -p user.info -t "$TAG" "Started in batch mode"
    main_proc
    logger -p user.info -t "$TAG" "Ended batch mode"
    exit 0
fi

exit 0
