#! /bin/sh
# $OpenLDAP$
## This work is part of OpenLDAP Software <http://www.openldap.org/>.
##
## Copyright 1998-2021 The OpenLDAP Foundation.
## All rights reserved.
##
## Redistribution and use in source and binary forms, with or without
## modification, are permitted only as authorized by the OpenLDAP
## Public License.
##
## A copy of this license is available in the file LICENSE in the
## top-level directory of the distribution or, alternatively, at
## <http://www.OpenLDAP.org/license.html>.

echo "running defines.sh"
. $SRCDIR/scripts/defines.sh

if test $SYNCPROV = syncprovno; then
	echo "Syncrepl provider overlay not available, test skipped"
	exit 0
fi
if test $ACCESSLOG = accesslogno; then
	echo "Accesslog overlay not available, test skipped"
	exit 0
fi
if test $BACKEND = ldif ; then
	echo "$BACKEND backend unsuitable, test skipped"
	exit 0
fi

dtest=`date +%N|sed s/...$//`

if test $dtest = N; then
	echo "nanosecond date values not supported, test skipped"
	exit 0
fi

# This mimics the scenario where a single server has been used until now (no
# syncprov either, so no contextCSN) and we convert it to a delta-MPR setup:
# 1. stop the server (note that there is likely no contextCSN in the DB at this point)
# 2. configure all servers to delta-replicate from each other and start them up
#    - empty servers will start with a refresh of the main DB
#    - when the refresh is successful they should change over to replicating the log
# 3. keep making changes on all servers to see things still work

echo "This test tracks a case where slapd deadlocks during a significant write load"
echo "See https://bugs.openldap.org/show_bug.cgi?id=8752 for more information."

MPR=4
iterations=20000
check_sync_every=100
MAPSIZE=`expr 100 \* 1024 \* 1024`
XDIR=$TESTDIR/srv

mkdir -p $TESTDIR

ITS=8752
ITSDIR=$DATADIR/regressions/its$ITS

n=1
while [ $n -le $MPR ]; do
	DBDIR=${XDIR}$n/db
	mkdir -p ${XDIR}$n $DBDIR.1 $DBDIR.2
	n=`expr $n + 1`
done

KILLPIDS=

echo "Starting slapd on TCP/IP port $PORT1..."
. $CONFFILTER $BACKEND < $ITSDIR/slapd.conf > $CONF1
$SLAPD -f $CONF1 -h $URI1 -d $LVL > $LOG1 2>&1 &
PID=$!
if test $WAIT != 0 ; then
	echo PID $PID
	read foo
fi
KILLPIDS="$PID"

sleep $SLEEP0

echo "Using ldapsearch to check that slapd is running..."
for i in 0 1 2 3 4 5; do
	$LDAPSEARCH -s base -b "$MONITOR" -H $URI1 \
		'objectclass=*' > /dev/null 2>&1
	RC=$?
	if test $RC = 0 ; then
		break
	fi
	echo "Waiting $SLEEP1 seconds for slapd to start..."
	sleep $SLEEP1
done

if test $RC != 0 ; then
	echo "ldapsearch failed ($RC)!"
	test $KILLSERVERS != no && kill -HUP $KILLPIDS
	exit $RC
fi

echo "Populating database on first provider..."
$LDAPADD -D $MANAGERDN -H $URI1 -w $PASSWD << EOMODS >> $TESTOUT 2>&1
dn: $BASEDN
objectClass: organization
objectClass: dcObject
o: Example, Inc.
dc: example

dn: ou=People,$BASEDN
objectClass: organizationalUnit
ou: People

dn: ou=Groups,$BASEDN
objectClass: organizationalUnit
ou: Groups

dn: cn=Roger Rabbit,ou=People,$BASEDN
objectClass: inetOrgPerson
cn: Roger Rabbit
sn: Rabbit

dn: cn=Baby Herman,ou=People,$BASEDN
objectClass: inetOrgPerson
cn: Baby Herman
sn: Herman

dn: cn=Jessica_Rabbit,ou=People,$BASEDN
objectClass: inetOrgPerson
cn: Jessica_Rabbit
sn: Rabbit

dn: cn=Bugs_Bunny,ou=People,$BASEDN
objectClass: inetOrgPerson
cn: Bugs_Bunny
sn: Bunny

dn: cn=Daffy_Duck,ou=People,$BASEDN
objectClass: inetOrgPerson
cn: Daffy_Duck
sn: Duck

dn: cn=Elmer_Fudd,ou=People,$BASEDN
objectClass: inetOrgPerson
cn: Elmer_Fudd
sn: Fudd

dn: cn=Cartoonia,ou=Groups,$BASEDN
objectClass: groupOfNames
cn: Cartoonia
member: cn=Roger Rabbit,ou=People,$BASEDN
member: cn=Baby Herman,ou=People,$BASEDN
EOMODS

RC=$?

if test $RC != 0 ; then
	echo "ldapadd failed ($RC)!"
	test $KILLSERVERS != no && kill -HUP $KILLPIDS
	exit $RC
fi

echo "Stopping slapd and reworking configuration for MPR..."

kill -HUP $KILLPIDS
wait $KILLPIDS

KILLPIDS=
n=1
while [ $n -le $MPR ]; do
	MYURI=`eval echo '$URI'$n`
	MYLOG=`eval echo '$LOG'$n`
	MYCONF=`eval echo '$CONF'$n`
	echo "Starting provider slapd on TCP/IP URI $MYURI"
	. $CONFFILTER $BACKEND < $ITSDIR/slapd.conf.mpr > $TESTDIR/slapd.conf
	sed -e "s/MPR/$n/g" -e "s/wronglog/log/" -e "s/@MAPSIZE@/$MAPSIZE/" $TESTDIR/slapd.conf > $MYCONF
	j=1
	while [ $j -le $MPR ]; do
		MMCURI=`eval echo '$URI'$j`
		sed -e "s|MMC${j}|${MMCURI}|" $MYCONF > $TESTDIR/slapd.conf
		mv $TESTDIR/slapd.conf $MYCONF
		j=`expr $j + 1`
	done
	if [ -f $TESTDIR/slapd.conf ]; then
		rm -f $TESTDIR/slapd.conf
	fi
	$SLAPD -f $MYCONF -h $MYURI  -d $LVL >> $MYLOG 2>&1 &
	PID=$!
	if test $WAIT != 0 ; then
		echo PID $PID
		read foo
	fi
	KILLPIDS="$PID $KILLPIDS"
	sleep $SLEEP1

	echo "Using ldapsearch to check that provider slapd is running..."
	for i in 0 1 2 3 4 5; do
		$LDAPSEARCH -s base -b "" -H $MYURI \
			'objectclass=*' > /dev/null 2>&1
		RC=$?
		if test $RC = 0 ; then
			break
		fi
		echo "Waiting $SLEEP1 seconds for slapd to start..."
		sleep $SLEEP1
	done

	if test $RC != 0 ; then
		echo "ldapsearch failed ($RC)!"
		test $KILLSERVERS != no && kill -HUP $KILLPIDS
		exit $RC
	fi
	n=`expr $n + 1`
done

echo "Setting up accesslog on each provider..."
n=1
while [ $n -le $MPR ]; do
	echo "Modifying dn: cn=Elmer_Fudd,ou=People,$BASEDN on provider $n"
	MYURI=`eval echo '$URI'$n`
	$LDAPMODIFY -v -D "$MANAGERDN" -H $MYURI -w $PASSWD > \
		$TESTOUT 2>&1 << EOMODS

dn: cn=Elmer_Fudd,ou=People,$BASEDN
changetype: modify
replace: sn
sn: Fudd
EOMODS
	RC=$?
	if test $RC != 0; then
		echo "ldapmodify failed ($RC)"
		test $KILLSERVERS != no && kill -HUP $KILLPIDS
		exit 1
	fi
	sleep $SLEEP1
	n=`expr $n + 1`
done

#echo "Letting server 1 establish its own contextCSN..."
#echo "Modifying dn: cn=Elmer_Fudd,ou=People,$BASEDN on $URI1"
#$LDAPMODIFY -v -D "$MANAGERDN" -H $URI1 -w $PASSWD > \
#    $TESTOUT 2>&1 << EOMODS
#dn: cn=Elmer_Fudd,ou=People,$BASEDN
#changetype: modify
#replace: sn
#sn: Fudd
#EOMODS

for i in 0 1 2 3 4 5; do
	j=1
	while [ $j -le $MPR ]; do
        MYURI=`eval echo '$URI'$j`
		$LDAPSEARCH -b "$BASEDN" -H "$MYURI" \
			'*' '+' >"$TESTDIR/server$j.out" 2>&1
		RC=$?
		if test $RC != 0 ; then
			echo "ldapsearch failed ($RC)!"
			test $KILLSERVERS != no && kill -HUP $KILLPIDS
			exit $RC
		fi
        $LDIFFILTER -s a < "$TESTDIR/server$j.out" > "$TESTDIR/server$j.flt"
		j=`expr $j + 1`
	done

	in_sync=1
	j=1
	while [ $j -lt $MPR ]; do
		k=$j
		j=`expr $j + 1`
		$CMP "$TESTDIR/server$k.flt" "$TESTDIR/server$j.flt" > $CMPOUT
		if test $? != 0 ; then
			in_sync=0
		fi
	done
	if test $in_sync = 1; then
		break
	fi

	echo "Waiting $SLEEP1 seconds for servers to catch up..."
	sleep $SLEEP1
done

if test $in_sync = 0; then
	echo "Servers did not replicate in time"
	test $KILLSERVERS != no && kill -HUP $KILLPIDS
	exit 1
fi

echo "The next step of the test will perform $iterations random write operations and may take some time."
echo "As this test is for a deadlock, it will take manual intervention to exit the test if one occurs."

echo "Starting random provider/entry modifications..."
DN1="cn=Elmer_Fudd,ou=People,$BASEDN"
VAL1="Fudd"

DN2="cn=Jessica_Rabbit,ou=People,$BASEDN"
VAL2="Rabbit"

DN3="cn=Bugs_Bunny,ou=People,$BASEDN"
VAL3="Bunny"

DN4="cn=Daffy_Duck,ou=People,$BASEDN"
VAL4="Duck"

n=1
while [ $n -le $iterations ]; do
	seed=`date +%N|sed s/...$//`
	rvalue=`echo|awk "BEGIN {srand($seed)
{print int(1+rand()*$MPR)}}"`
	MYURI=`eval echo '$URI'$rvalue`
	seed=`date +%N|sed s/...$//`
	rvalue=`echo|awk "BEGIN {srand($seed)
{print int(1+rand()*4)}}"`
	MYDN=`eval echo '$DN'$rvalue`
	MYVAL=`eval echo '$VAL'$rvalue`
	echo "Modifying $MYURI entry $MYDN with value $MYVAL iteration $n of $iterations"
	$LDAPMODIFY -v -D "$MANAGERDN" -H $MYURI -w $PASSWD > \
		$TESTOUT 2>&1 << EOMODS

dn: $MYDN
changetype: modify
replace: sn
sn: $MYVAL
EOMODS
	RC=$?
	if test $RC != 0; then
		echo "ldapmodify failed ($RC)"
		test $KILLSERVERS != no && kill -HUP $KILLPIDS
		exit 1
	fi

	if [ "$check_sync_every" -gt 0 ] && [ `expr $n % $check_sync_every` = 0 ]; then
		i=1
        echo "Checking replication status before we start iteration $n..."
        for i in 0 1 2 3 4 5; do
            j=1
            while [ $j -le $MPR ]; do
                MYURI=`eval echo '$URI'$j`
                echo "Reading database from server $j..."
                $LDAPSEARCH -b "$BASEDN" -H "$MYURI" \
                    '*' '+' >"$TESTDIR/server$j.out" 2>&1
                RC=$?
                if test $RC != 0 ; then
                    echo "ldapsearch failed ($RC)!"
                    test $KILLSERVERS != no && kill -HUP $KILLPIDS
                    exit $RC
                fi
                $LDIFFILTER -s a < "$TESTDIR/server$j.out" > "$TESTDIR/server$j.flt"
                j=`expr $j + 1`
            done

            in_sync=1
            j=1
            while [ $j -lt $MPR ]; do
                k=`expr $j + 1`
                $CMP "$TESTDIR/server$j.flt" "$TESTDIR/server$k.flt" > $CMPOUT
                if test $? != 0 ; then
                    in_sync=0
                fi
                j=$k
            done
            if test $in_sync = 1; then
                break
            fi

            echo "Waiting $SLEEP1 seconds for servers to catch up..."
            sleep $SLEEP1
        done

        if test $in_sync = 0; then
            echo "Servers did not replicate in time"
            test $KILLSERVERS != no && kill -HUP $KILLPIDS
            exit 1
        fi
    fi
    n=`expr $n + 1`
done

kill -HUP $KILLPIDS
wait $KILLPIDS

echo "The next step of the test will perform $iterations random write operations and may take some time."
echo "As this test is for a deadlock, it will take manual intervention to exit the test if one occurs."

echo "Starting servers again, this time with the wrong logbase setting..."
KILLPIDS=
n=1
while [ $n -le $MPR ]; do
    MYURI=`eval echo '$URI'$n`
    MYLOG=`eval echo '$LOG'$n`
    MYCONF=`eval echo '$CONF'$n`
    echo "Starting provider slapd on TCP/IP URI $MYURI"
    . $CONFFILTER $BACKEND < $ITSDIR/slapd.conf.mpr > $TESTDIR/slapd.conf
    sed -e "s/MPR/$n/g" -e "s/@MAPSIZE@/$MAPSIZE/" $TESTDIR/slapd.conf > $MYCONF
    j=1
    while [ $j -le $MPR ]; do
        MMCURI=`eval echo '$URI'$j`
        sed -e "s|MMC${j}|${MMCURI}|" $MYCONF > $TESTDIR/slapd.conf
        mv $TESTDIR/slapd.conf $MYCONF
        j=`expr $j + 1`
    done
    if [ -f $TESTDIR/slapd.conf ]; then
        rm -f $TESTDIR/slapd.conf
    fi
    $SLAPD -f $MYCONF -h $MYURI  -d $LVL >> $MYLOG 2>&1 &
    PID=$!
    if test $WAIT != 0 ; then
        echo PID $PID
        read foo
    fi
    KILLPIDS="$PID $KILLPIDS"
    sleep $SLEEP1

    echo "Using ldapsearch to check that provider slapd is running..."
    for i in 0 1 2 3 4 5; do
        $LDAPSEARCH -s base -b "" -H $MYURI \
            'objectclass=*' > /dev/null 2>&1
                    RC=$?
                    if test $RC = 0 ; then
                        break
                    fi
                    echo "Waiting $SLEEP1 seconds for slapd to start..."
                    sleep $SLEEP1
                done

                if test $RC != 0 ; then
                    echo "ldapsearch failed ($RC)!"
                    test $KILLSERVERS != no && kill -HUP $KILLPIDS
                    exit $RC
                fi
                n=`expr $n + 1`
            done

            echo "Starting random provider/entry modifications..."
            n=1
            while [ $n -le $iterations ]; do
                seed=`date +%N|sed s/...$//`
                rvalue=`echo|awk "BEGIN {srand($seed)
                {print int(1+rand()*$MPR)}}"`
                    MYURI=`eval echo '$URI'$rvalue`
                    seed=`date +%N|sed s/...$//`
                    rvalue=`echo|awk "BEGIN {srand($seed)
                    {print int(1+rand()*4)}}"`
                        MYDN=`eval echo '$DN'$rvalue`
                        MYVAL=`eval echo '$VAL'$rvalue`
                        echo "Modifying $MYURI entry $MYDN with value $MYVAL iteration $n of $iterations"
                        $LDAPMODIFY -v -D "$MANAGERDN" -H $MYURI -w $PASSWD > \
                            $TESTOUT 2>&1 << EOMODS

dn: $MYDN
changetype: modify
replace: sn
sn: $MYVAL
EOMODS
RC=$?
if test $RC != 0; then
    echo "ldapmodify failed ($RC)"
    test $KILLSERVERS != no && kill -HUP $KILLPIDS
    exit 1
fi

if [ "$check_sync_every" -gt 0 ] && [ `expr $n % $check_sync_every` = 0 ]; then
    i=1
    echo "Checking replication status before we start iteration $n..."
    for i in 0 1 2 3 4 5; do
        j=1
        while [ $j -le $MPR ]; do
            MYURI=`eval echo '$URI'$j`
            echo "Reading database from server $j..."
            $LDAPSEARCH -b "$BASEDN" -H "$MYURI" \
                '*' '+' >"$TESTDIR/server$j.out" 2>&1
                            RC=$?
                            if test $RC != 0 ; then
                                echo "ldapsearch failed ($RC)!"
                                test $KILLSERVERS != no && kill -HUP $KILLPIDS
                                exit $RC
                            fi
                $LDIFFILTER -s a < "$TESTDIR/server$j.out" > "$TESTDIR/server$j.flt"
                j=`expr $j + 1`
            done

            in_sync=1
            j=1
            while [ $j -lt $MPR ]; do
                k=`expr $j + 1`
                $CMP "$TESTDIR/server$j.flt" "$TESTDIR/server$k.flt" > $CMPOUT
                if test $? != 0 ; then
                    in_sync=0
                fi
                j=$k
            done
            if test $in_sync = 1; then
                break
            fi

            echo "Waiting $SLEEP1 seconds for servers to catch up..."
            sleep $SLEEP1
        done

        if test $in_sync = 0; then
            echo "Servers did not replicate in time"
            test $KILLSERVERS != no && kill -HUP $KILLPIDS
            exit 1
        fi
	fi
	n=`expr $n + 1`
done

test $KILLSERVERS != no && kill -HUP $KILLPIDS

echo ">>>>> Test succeeded"

test $KILLSERVERS != no && wait

exit 0