I have a tomcat server running a spring boot application on a host using CentOS 8 that connects to a mail client using SMTP. Problem is it abruptly closes connection before TLS handshake occurs with a STARTTLS option. It works OK with other nodes running CentOS 7 and Oracle linux server. The problem cannot be with the application otherwise it would not work on any nodes. I checked using Wireshark to see that it was our application node terminating the connection. Here is the TCP communication transcript I recorded using tcpdump.
220 mta1.***.local ESMTP Postfix
EHLO ****
250-mta1.***.local
250-PIPELINING
250-SIZE 69730304
250-VRFY
250-ETRN
250-STARTTLS
250-ENHANCEDSTATUSCODES
250-8BITMIME
250 DSN
STARTTLS
220 2.0.0 Ready to start TLS
And nothing after that. Connection termination was abrupt, no timeout or anything like that. I also checked for server.xml file for any settings there was no mention of mail or smtp in any way.
I tried the same using openssl and it works, successfully starting TLS over smtp connection so it is not the issue with server not supporting the option either. Does anyone have any idea why this might have happened?
openssl s_client -connect smtp.****.gov.**:587 -starttls smtp
My catalina.sh file
#!/bin/sh
cygwin=false
darwin=false
os400=false
hpux=false
case "`uname`" in
CYGWIN*) cygwin=true;;
Darwin*) darwin=true;;
OS400*) os400=true;;
HP-UX*) hpux=true;;
esac
PRG="$0"
while [ -h "$PRG" ]; do
ls=`ls -ld "$PRG"`
link=`expr "$ls" : '.*-> \(.*\)$'`
if expr "$link" : '/.*' > /dev/null; then
PRG="$link"
else
PRG=`dirname "$PRG"`/"$link"
fi
done
PRGDIR=`dirname "$PRG"`
[ -z "$CATALINA_HOME" ] && CATALINA_HOME=`cd "$PRGDIR/.." >/dev/null; pwd`
[ -z "$CATALINA_BASE" ] && CATALINA_BASE="$CATALINA_HOME"
CLASSPATH=
if [ -r "$CATALINA_BASE/bin/setenv.sh" ]; then
. "$CATALINA_BASE/bin/setenv.sh"
elif [ -r "$CATALINA_HOME/bin/setenv.sh" ]; then
. "$CATALINA_HOME/bin/setenv.sh"
fi
if $cygwin; then
[ -n "$JAVA_HOME" ] && JAVA_HOME=`cygpath --unix "$JAVA_HOME"`
[ -n "$JRE_HOME" ] && JRE_HOME=`cygpath --unix "$JRE_HOME"`
[ -n "$CATALINA_HOME" ] && CATALINA_HOME=`cygpath --unix "$CATALINA_HOME"`
[ -n "$CATALINA_BASE" ] && CATALINA_BASE=`cygpath --unix "$CATALINA_BASE"`
[ -n "$CLASSPATH" ] && CLASSPATH=`cygpath --path --unix "$CLASSPATH"`
fi
case $CATALINA_HOME in
*:*) echo "Using CATALINA_HOME: $CATALINA_HOME";
echo "Unable to start as CATALINA_HOME contains a colon (:) character";
exit 1;
esac
case $CATALINA_BASE in
*:*) echo "Using CATALINA_BASE: $CATALINA_BASE";
echo "Unable to start as CATALINA_BASE contains a colon (:) character";
exit 1;
esac
if $os400; then
COMMAND='chgjob job('$JOBNAME') runpty(6)'
system $COMMAND
export QIBM_MULTI_THREADED=Y
fi
if $os400; then
. "$CATALINA_HOME"/bin/setclasspath.sh
else
if [ -r "$CATALINA_HOME"/bin/setclasspath.sh ]; then
. "$CATALINA_HOME"/bin/setclasspath.sh
else
echo "Cannot find $CATALINA_HOME/bin/setclasspath.sh"
echo "This file is needed to run this program"
exit 1
fi
fi
if [ ! -z "$CLASSPATH" ] ; then
CLASSPATH="$CLASSPATH":
fi
CLASSPATH="$CLASSPATH""$CATALINA_HOME"/bin/bootstrap.jar
if [ -z "$CATALINA_OUT" ] ; then
CATALINA_OUT=/home/******/logs/catalinalogs/catalina.out
fi
if [ -z "$CATALINA_TMPDIR" ] ; then
CATALINA_TMPDIR="$CATALINA_BASE"/temp
fi
if [ -r "$CATALINA_BASE/bin/tomcat-juli.jar" ] ; then
CLASSPATH=$CLASSPATH:$CATALINA_BASE/bin/tomcat-juli.jar
else
CLASSPATH=$CLASSPATH:$CATALINA_HOME/bin/tomcat-juli.jar
fi
have_tty=0
if [ -t 0 ]; then
have_tty=1
fi
if $cygwin; then
JAVA_HOME=`cygpath --absolute --windows "$JAVA_HOME"`
JRE_HOME=`cygpath --absolute --windows "$JRE_HOME"`
CATALINA_HOME=`cygpath --absolute --windows "$CATALINA_HOME"`
CATALINA_BASE=`cygpath --absolute --windows "$CATALINA_BASE"`
CATALINA_TMPDIR=`cygpath --absolute --windows "$CATALINA_TMPDIR"`
CLASSPATH=`cygpath --path --windows "$CLASSPATH"`
[ -n "$JAVA_ENDORSED_DIRS" ] && JAVA_ENDORSED_DIRS=`cygpath --path --windows "$JAVA_ENDORSED_DIRS"`
fi
if [ -z "$JSSE_OPTS" ] ; then
JSSE_OPTS="-Djdk.tls.ephemeralDHKeySize=2048"
fi
JAVA_OPTS="$JAVA_OPTS $JSSE_OPTS"
JAVA_OPTS="$JAVA_OPTS -Djava.protocol.handler.pkgs=org.apache.catalina.webresources"
if [ -z "$LOGGING_CONFIG" ]; then
if [ -r "$CATALINA_BASE"/conf/logging.properties ]; then
LOGGING_CONFIG="-Djava.util.logging.config.file=$CATALINA_BASE/conf/logging.properties"
else
LOGGING_CONFIG="-Dnop"
fi
fi
if [ -z "$LOGGING_MANAGER" ]; then
LOGGING_MANAGER="-Djava.util.logging.manager=org.apache.juli.ClassLoaderLogManager"
fi
if [ -z "$UMASK" ]; then
UMASK="0027"
fi
umask $UMASK
ENDORSED_PROP=ignore.endorsed.dirs
if [ -n "$JAVA_ENDORSED_DIRS" ]; then
ENDORSED_PROP=java.endorsed.dirs
fi
if [ -d "$CATALINA_HOME/endorsed" ]; then
ENDORSED_PROP=java.endorsed.dirs
fi
JAVA_OPTS="$JAVA_OPTS -Dorg.apache.catalina.security.SecurityListener.UMASK=`umask`"
if [ -z "$USE_NOHUP" ]; then
if $hpux; then
USE_NOHUP="true"
else
USE_NOHUP="false"
fi
fi
unset _NOHUP
if [ "$USE_NOHUP" = "true" ]; then
_NOHUP="nohup"
fi
JDK_JAVA_OPTIONS="$JDK_JAVA_OPTIONS --add-opens=java.base/java.lang=ALL-UNNAMED"
JDK_JAVA_OPTIONS="$JDK_JAVA_OPTIONS --add-opens=java.base/java.io=ALL-UNNAMED"
JDK_JAVA_OPTIONS="$JDK_JAVA_OPTIONS --add-opens=java.rmi/sun.rmi.transport=ALL-UNNAMED"
export JDK_JAVA_OPTIONS
if [ $have_tty -eq 1 ]; then
echo "Using CATALINA_BASE: $CATALINA_BASE"
echo "Using CATALINA_HOME: $CATALINA_HOME"
echo "Using CATALINA_TMPDIR: $CATALINA_TMPDIR"
if [ "$1" = "debug" ] ; then
echo "Using JAVA_HOME: $JAVA_HOME"
else
echo "Using JRE_HOME: $JRE_HOME"
fi
echo "Using CLASSPATH: $CLASSPATH"
if [ ! -z "$CATALINA_PID" ]; then
echo "Using CATALINA_PID: $CATALINA_PID"
fi
fi
if [ "$1" = "jpda" ] ; then
if [ -z "$JPDA_TRANSPORT" ]; then
JPDA_TRANSPORT="dt_socket"
fi
if [ -z "$JPDA_ADDRESS" ]; then
JPDA_ADDRESS="localhost:8000"
fi
if [ -z "$JPDA_SUSPEND" ]; then
JPDA_SUSPEND="n"
fi
if [ -z "$JPDA_OPTS" ]; then
JPDA_OPTS="-agentlib:jdwp=transport=$JPDA_TRANSPORT,address=$JPDA_ADDRESS,server=y,suspend=$JPDA_SUSPEND"
fi
CATALINA_OPTS="$JPDA_OPTS $CATALINA_OPTS"
shift
fi
if [ "$1" = "debug" ] ; then
if $os400; then
echo "Debug command not available on OS400"
exit 1
else
shift
if [ "$1" = "-security" ] ; then
if [ $have_tty -eq 1 ]; then
echo "Using Security Manager"
fi
shift
exec "$_RUNJDB" "$LOGGING_CONFIG" $LOGGING_MANAGER $JAVA_OPTS $CATALINA_OPTS \
-D$ENDORSED_PROP="$JAVA_ENDORSED_DIRS" \
-classpath "$CLASSPATH" \
-sourcepath "$CATALINA_HOME"/../../java \
-Djava.security.manager \
-Djava.security.policy=="$CATALINA_BASE"/conf/catalina.policy \
-Dcatalina.base="$CATALINA_BASE" \
-Dcatalina.home="$CATALINA_HOME" \
-Djava.io.tmpdir="$CATALINA_TMPDIR" \
org.apache.catalina.startup.Bootstrap "$@" start
else
exec "$_RUNJDB" "$LOGGING_CONFIG" $LOGGING_MANAGER $JAVA_OPTS $CATALINA_OPTS \
-D$ENDORSED_PROP="$JAVA_ENDORSED_DIRS" \
-classpath "$CLASSPATH" \
-sourcepath "$CATALINA_HOME"/../../java \
-Dcatalina.base="$CATALINA_BASE" \
-Dcatalina.home="$CATALINA_HOME" \
-Djava.io.tmpdir="$CATALINA_TMPDIR" \
org.apache.catalina.startup.Bootstrap "$@" start
fi
fi
elif [ "$1" = "run" ]; then
shift
if [ "$1" = "-security" ] ; then
if [ $have_tty -eq 1 ]; then
echo "Using Security Manager"
fi
shift
eval exec "\"$_RUNJAVA\"" "\"$LOGGING_CONFIG\"" $LOGGING_MANAGER "$JAVA_OPTS" "$CATALINA_OPTS" \
-D$ENDORSED_PROP="\"$JAVA_ENDORSED_DIRS\"" \
-classpath "\"$CLASSPATH\"" \
-Djava.security.manager \
-Djava.security.policy=="\"$CATALINA_BASE/conf/catalina.policy\"" \
-Dcatalina.base="\"$CATALINA_BASE\"" \
-Dcatalina.home="\"$CATALINA_HOME\"" \
-Djava.io.tmpdir="\"$CATALINA_TMPDIR\"" \
org.apache.catalina.startup.Bootstrap "$@" start
else
eval exec "\"$_RUNJAVA\"" "\"$LOGGING_CONFIG\"" $LOGGING_MANAGER "$JAVA_OPTS" "$CATALINA_OPTS" \
-D$ENDORSED_PROP="\"$JAVA_ENDORSED_DIRS\"" \
-classpath "\"$CLASSPATH\"" \
-Dcatalina.base="\"$CATALINA_BASE\"" \
-Dcatalina.home="\"$CATALINA_HOME\"" \
-Djava.io.tmpdir="\"$CATALINA_TMPDIR\"" \
org.apache.catalina.startup.Bootstrap "$@" start
fi
elif [ "$1" = "start" ] ; then
if [ ! -z "$CATALINA_PID" ]; then
if [ -f "$CATALINA_PID" ]; then
if [ -s "$CATALINA_PID" ]; then
echo "Existing PID file found during start."
if [ -r "$CATALINA_PID" ]; then
PID=`cat "$CATALINA_PID"`
ps -p $PID >/dev/null 2>&1
if [ $? -eq 0 ] ; then
echo "Tomcat appears to still be running with PID $PID. Start aborted."
echo "If the following process is not a Tomcat process, remove the PID file and try again:"
ps -f -p $PID
exit 1
else
echo "Removing/clearing stale PID file."
rm -f "$CATALINA_PID" >/dev/null 2>&1
if [ $? != 0 ]; then
if [ -w "$CATALINA_PID" ]; then
cat /dev/null > "$CATALINA_PID"
else
echo "Unable to remove or clear stale PID file. Start aborted."
exit 1
fi
fi
fi
else
echo "Unable to read PID file. Start aborted."
exit 1
fi
else
rm -f "$CATALINA_PID" >/dev/null 2>&1
if [ $? != 0 ]; then
if [ ! -w "$CATALINA_PID" ]; then
echo "Unable to remove or write to empty PID file. Start aborted."
exit 1
fi
fi
fi
fi
fi
shift
touch "$CATALINA_OUT"
if [ "$1" = "-security" ] ; then
if [ $have_tty -eq 1 ]; then
echo "Using Security Manager"
fi
shift
eval $_NOHUP "\"$_RUNJAVA\"" "\"$LOGGING_CONFIG\"" $LOGGING_MANAGER "$JAVA_OPTS" "$CATALINA_OPTS" \
-D$ENDORSED_PROP="\"$JAVA_ENDORSED_DIRS\"" \
-classpath "\"$CLASSPATH\"" \
-Djava.security.manager \
-Djava.security.policy=="\"$CATALINA_BASE/conf/catalina.policy\"" \
-Dcatalina.base="\"$CATALINA_BASE\"" \
-Dcatalina.home="\"$CATALINA_HOME\"" \
-Djava.io.tmpdir="\"$CATALINA_TMPDIR\"" \
org.apache.catalina.startup.Bootstrap "$@" start \
>> "$CATALINA_OUT" 2>&1 "&"
else
eval $_NOHUP "\"$_RUNJAVA\"" "\"$LOGGING_CONFIG\"" $LOGGING_MANAGER "$JAVA_OPTS" "$CATALINA_OPTS" \
-D$ENDORSED_PROP="\"$JAVA_ENDORSED_DIRS\"" \
-classpath "\"$CLASSPATH\"" \
-Dcatalina.base="\"$CATALINA_BASE\"" \
-Dcatalina.home="\"$CATALINA_HOME\"" \
-Djava.io.tmpdir="\"$CATALINA_TMPDIR\"" \
org.apache.catalina.startup.Bootstrap "$@" start \
>> "$CATALINA_OUT" 2>&1 "&"
fi
if [ ! -z "$CATALINA_PID" ]; then
echo $! > "$CATALINA_PID"
fi
echo "Tomcat started."
elif [ "$1" = "stop" ] ; then
shift
SLEEP=5
if [ ! -z "$1" ]; then
echo $1 | grep "[^0-9]" >/dev/null 2>&1
if [ $? -gt 0 ]; then
SLEEP=$1
shift
fi
fi
FORCE=0
if [ "$1" = "-force" ]; then
shift
FORCE=1
fi
if [ ! -z "$CATALINA_PID" ]; then
if [ -f "$CATALINA_PID" ]; then
if [ -s "$CATALINA_PID" ]; then
kill -0 `cat "$CATALINA_PID"` >/dev/null 2>&1
if [ $? -gt 0 ]; then
echo "PID file found but either no matching process was found or the current user does not have permission to stop the process. Stop aborted."
exit 1
fi
else
echo "PID file is empty and has been ignored."
fi
else
echo "\$CATALINA_PID was set but the specified file does not exist. Is Tomcat running? Stop aborted."
exit 1
fi
fi
eval "\"$_RUNJAVA\"" $LOGGING_MANAGER "$JAVA_OPTS" \
-D$ENDORSED_PROP="\"$JAVA_ENDORSED_DIRS\"" \
-classpath "\"$CLASSPATH\"" \
-Dcatalina.base="\"$CATALINA_BASE\"" \
-Dcatalina.home="\"$CATALINA_HOME\"" \
-Djava.io.tmpdir="\"$CATALINA_TMPDIR\"" \
org.apache.catalina.startup.Bootstrap "$@" stop
if [ $? != 0 ]; then
if [ ! -z "$CATALINA_PID" ]; then
echo "The stop command failed. Attempting to signal the process to stop through OS signal."
kill -15 `cat "$CATALINA_PID"` >/dev/null 2>&1
fi
fi
if [ ! -z "$CATALINA_PID" ]; then
if [ -f "$CATALINA_PID" ]; then
while [ $SLEEP -ge 0 ]; do
kill -0 `cat "$CATALINA_PID"` >/dev/null 2>&1
if [ $? -gt 0 ]; then
rm -f "$CATALINA_PID" >/dev/null 2>&1
if [ $? != 0 ]; then
if [ -w "$CATALINA_PID" ]; then
cat /dev/null > "$CATALINA_PID"
FORCE=0
else
echo "The PID file could not be removed or cleared."
fi
fi
echo "Tomcat stopped."
break
fi
if [ $SLEEP -gt 0 ]; then
sleep 1
fi
if [ $SLEEP -eq 0 ]; then
echo "Tomcat did not stop in time."
if [ $FORCE -eq 0 ]; then
echo "PID file was not removed."
fi
echo "To aid diagnostics a thread dump has been written to standard out."
kill -3 `cat "$CATALINA_PID"`
fi
SLEEP=`expr $SLEEP - 1 `
done
fi
fi
KILL_SLEEP_INTERVAL=5
if [ $FORCE -eq 1 ]; then
if [ -z "$CATALINA_PID" ]; then
echo "Kill failed: \$CATALINA_PID not set"
else
if [ -f "$CATALINA_PID" ]; then
PID=`cat "$CATALINA_PID"`
echo "Killing Tomcat with the PID: $PID"
kill -9 $PID
while [ $KILL_SLEEP_INTERVAL -ge 0 ]; do
kill -0 `cat "$CATALINA_PID"` >/dev/null 2>&1
if [ $? -gt 0 ]; then
rm -f "$CATALINA_PID" >/dev/null 2>&1
if [ $? != 0 ]; then
if [ -w "$CATALINA_PID" ]; then
cat /dev/null > "$CATALINA_PID"
else
echo "The PID file could not be removed."
fi
fi
echo "The Tomcat process has been killed."
break
fi
if [ $KILL_SLEEP_INTERVAL -gt 0 ]; then
sleep 1
fi
KILL_SLEEP_INTERVAL=`expr $KILL_SLEEP_INTERVAL - 1 `
done
if [ $KILL_SLEEP_INTERVAL -lt 0 ]; then
echo "Tomcat has not been killed completely yet. The process might be waiting on some system call or might be UNINTERRUPTIBLE."
fi
fi
fi
fi
elif [ "$1" = "configtest" ] ; then
eval "\"$_RUNJAVA\"" $LOGGING_MANAGER "$JAVA_OPTS" \
-D$ENDORSED_PROP="\"$JAVA_ENDORSED_DIRS\"" \
-classpath "\"$CLASSPATH\"" \
-Dcatalina.base="\"$CATALINA_BASE\"" \
-Dcatalina.home="\"$CATALINA_HOME\"" \
-Djava.io.tmpdir="\"$CATALINA_TMPDIR\"" \
org.apache.catalina.startup.Bootstrap configtest
result=$?
if [ $result -ne 0 ]; then
echo "Configuration error detected!"
fi
exit $result
elif [ "$1" = "version" ] ; then
"$_RUNJAVA" \
-classpath "$CATALINA_HOME/lib/catalina.jar" \
org.apache.catalina.util.ServerInfo
else
echo "Usage: catalina.sh ( commands ... )"
echo "commands:"
if $os400; then
echo " debug Start Catalina in a debugger (not available on OS400)"
echo " debug -security Debug Catalina with a security manager (not available on OS400)"
else
echo " debug Start Catalina in a debugger"
echo " debug -security Debug Catalina with a security manager"
fi
echo " jpda start Start Catalina under JPDA debugger"
echo " run Start Catalina in the current window"
echo " run -security Start in the current window with security manager"
echo " start Start Catalina in a separate window"
echo " start -security Start in a separate window with security manager"
echo " stop Stop Catalina, waiting up to 5 seconds for the process to end"
echo " stop n Stop Catalina, waiting up to n seconds for the process to end"
echo " stop -force Stop Catalina, wait up to 5 seconds and then use kill -KILL if still running"
echo " stop n -force Stop Catalina, wait up to n seconds and then use kill -KILL if still running"
echo " configtest Run a basic syntax check on server.xml - check exit code for result"
echo " version What version of tomcat are you running?"
echo "Note: Waiting for the process to end and use of the -force option require that \$CATALINA_PID is defined"
exit 1
fi
My setenv.sh file
export JAVA_OPTS="$JAVA_OPTS -Xms3072m -Xmx3072m"