AZ Asia-Pacific - Zscaler Help Centre
All Categories Zscaler Private Access ZPA App Connector : Troubleshooting Script

ZPA App Connector : Troubleshooting Script

When troubleshooting ZPA App Connectors, it is worth considering all the changes mentioned in the article "ZPA App Connector : Ensuring Best Performance" - as well as the CPU/Memory/Disk allocated to the VM/OS. You can also look at the TLS connections/second to ensure the CPU’s are capable of processing the volume of mTunnels created during application access.

The script below can be run on a ZPA App Connector to pull all the necessary parameters to investigate offline. Mass-executing this across connectors would enable diagnostics to check for congruent system configuration and identification of any connector “hotspots” or “notspots” which could be addressed through policy/configuration.

ZPA App Connector - Troubleshooting Script:

#!/usr/bin/bash

#scp zpa-diag.sh connector.domain.com:/tmp

#ssh -t connector.domain.com '/tmp/zpa-diag.sh'

#scp 'connector.domain.com:/tmp/zpa-diag*.tar.gz' ./

#Some commands need ROOT - run this script as root, or sudo ./zpa-diag.sh

#or run visudo and add the following line to the end of the sudoers file

#admin ALL=(ALL) NOPASSWD: /usr/sbin/lsof, /usr/sbin/ss, /usr/bin/openssl

exec 3>&2

exec 2> /dev/null

echo Creating Diagnostics Directory

mkdir /tmp/zpa-diag

#Following commands require ROOT. Either run script as root, or edit SUDOERS as above

#If you run script as root, remove sudo commands below

CID=$(sudo openssl x509 -subject -noout -in /opt/zscaler/var/cert.pem | cut -d '=' -f 4 | cut -d '-' -f 2 | cut -d '.' -f 1)

sudo lsof -n -P > /tmp/zpa-diag/lsof-output.txt

sudo lsof -n | wc -l >/tmp/zpa-diag/lsof-opencount.txt

sudo ss -s > /tmp/zpa-diag/ss.txt

echo Connector ID = $CID

echo $CID >> /tmp/zpa-diag/$CID

echo Collecting AWS Instance Type

curl --connect-timeout 2 http://169.254.169.254/latest/meta-data/instance-type -o /tmp/zpa-diag/instance-type

curl --connect-timeout 2 http://169.254.169.254/latest/meta-data/placement/availability-zone -o /tmp/zpa-diag/availability-zone

echo Collecting Azure Instance Type

curl --connect-timeout 2 -H metadata:true "http://169.254.169.254/metadata/instance/compute/vmSize?api-version=2017-08-01&format=text" -o /tmp/zpa-diag/azure-instance-type

curl --connect-timeout 2 -H metadata:true "http://169.254.169.254/metadata/instance/compute/location?api-version=2017-08-01&format=text" -o /tmp/zpa-diag/azure-availabilty-zone

echo Running Openssl Checks

echo openssl speed -evp aes-256-cbc > /tmp/zpa-diag/openssl.txt

openssl speed -evp aes-256-cbc >> /tmp/zpa-diag/openssl.txt

echo >> /tmp/zpa-diag/openssl.txt

echo openssl speed aes-256-cbc >> /tmp/zpa-diag/openssl.txt

openssl speed aes-256-cbc >> /tmp/zpa-diag/openssl.txt

echo Collecting Journal

journalctl > /tmp/zpa-diag/journal.log

journalctl -u zpa-connector -S -1m | grep Mtunnels >/tmp/zpa-diag/mtunnels.txt

echo Collecting CPU/Memory Info

echo Memory Report

date >> memory_report.txt

echo >> /tmp/zpa-diag/memory_report.txt

echo UNAME >> /tmp/zpa-diag/memory_report.txt

uname -a >> /tmp/zpa-diag/memory_report.txt

echo >> /tmp/zpa-diag/memory_report.txt

echo HOSTNAME >> /tmp/zpa-diag/memory_report.txt

hostname >> /tmp/zpa-diag/memory_report.txt

echo >> /tmp/zpa-diag/memory_report.txt

echo LSCPU >> /tmp/zpa-diag/memory_report.txt

lscpu >> /tmp/zpa-diag/memory_report.txt

echo >> /tmp/zpa-diag/memory_report.txt

echo /PROC/CPUINFO >> /tmp/zpa-diag/memory_report.txt

cat /proc/cpuinfo >> /tmp/zpa-diag/memory_report.txt

echo >> /tmp/zpa-diag/memory_report.txt

echo /PROC/MEMINFO >> /tmp/zpa-diag/memory_report.txt

cat /proc/meminfo >> /tmp/zpa-diag/memory_report.txt

echo >> /tmp/zpa-diag/memory_report.txt

echo Processes >> /tmp/zpa-diag/memory_report.txt

echo "ps aux --sort=-pmem | head -5" >> /tmp/zpa-diag/memory_report.txt

ps aux --sort=-pmem | head -5 >> /tmp/zpa-diag/memory_report.txt

echo >> /tmp/zpa-diag/memory_report.txt

echo "curl -s 127.0.0.1:9000/memory/status" >> /tmp/zpa-diag/memory_report.txt

curl -s 127.0.0.1:9000/memory/status >> /tmp/zpa-diag/memory_report.txt

echo >> /tmp/zpa-diag/memory_report.txt

echo "curl -s 127.0.0.1:9000/memory/argo" >> /tmp/zpa-diag/memory_report.txt

curl -s 127.0.0.1:9000/memory/argo >> /tmp/zpa-diag/memory_report.txt

echo Collecting File Descriptors

echo sysctl fs.file-max > /tmp/zpa-diag/file_descriptors.txt

sysctl fs.file-max >> /tmp/zpa-diag/file_descriptors.txt

echo >> /tmp/zpa-diag/file_descriptors.txt

echo ulimit -Hn >> /tmp/zpa-diag/file_descriptors.txt

ulimit -Hn >> /tmp/zpa-diag/file_descriptors.txt

echo >> /tmp/zpa-diag/file_descriptors.txt

echo ulimit -Sn >> /tmp/zpa-diag/file_descriptors.txt

ulimit -Sn

echo Collecting Disk Utilisation

echo DISK Utilisation >> /tmp/zpa-diag/disk_report.txt

df -h >> /tmp/zpa-diag/disk_report.txt

echo Collecting Port Range

mkdir /tmp/zpa-diag/portrange

cp /proc/sys/net/ipv4/iplocalport_range >> /tmp/zpa-diag/portrange

cp /etc/sysctl.conf >> /tmp/zpa-diag/portrange

cp /etc/sysctl.d/* /tmp/zpa-diag/portrange

sysctl net.ipv4.iplocalport_range >> /tmp/zpa-diag/portrange/current

echo Resolving co2br.prod.zpath.net - performing MTR

echo resolved IPs

dig co2br.prod.zpath.net | grep "IN A" | cut -f 3

for x in $(dig co2br.prod.zpath.net | grep "IN A" | cut -f 3)

do

echo MTR to $x

mtr -rnc5 $x > /tmp/zpa-diag/mtr-$x.txt

done

cp /etc/resolv.conf /tmp/zpa-diag

cp /etc/hosts /tmp/zpa-diag

echo Collecting ZPA Statistics

curl '127.0.0.1:9000/debug' >> /tmp/zpa-diag/connectordebugstate.txt

curl '127.0.0.1:9000/assistant/dns/state/dump' >> /tmp/zpa-diag/connectordnsstate_dump.txt

curl '127.0.0.1:9000/assistant/app/dump/statesummary' >> /tmp/zpa-diag/connectorappstatesummary.txt

curl '127.0.0.1:9000/assistant/data/mtunnel/dump/stats' >> /tmp/zpa-diag/connectormtunnelstats.txt

ls -lR /opt/zscaler/ >> /tmp/zpa-diag/dir.txt

cp /opt/zscaler/var/version /tmp/zpa-diag

cp /opt/zscaler/var/updater.version /tmp/zpa-diag

uptime >> /tmp/zpa-diag/uptime.txt

tar -zcvf /tmp/zpa-diag-$CID.tar.gz /tmp/zpa-diag/*

rm -rf /tmp/zpa-diag