Page MenuHomeGitPull.it

How to destroy a production Debian server
ActivePublic

Authored by valerio.bozzolan on Jun 29 2021, 18:26.
Tags
None
Referenced Files
F1492119: Screenshot_20210629_183245.png
Jun 29 2021, 21:12
F1492116: How to destroy a Debian server
Jun 29 2021, 18:26
Subscribers
None
cd
apt install pdns
apt install powerdns powerdns-recursor powerdns-backend-mysql
apt-cache search powerdns
apt install pdns-server pdns-tools pdns-backend-mydns pdns-recursor
apt install pdns-server pdns-tools pdns-backend-mydns
journalctl -xe
apt-cache search pdns
apt-get install pdns-recursor
cd /etc/
ls | grep pdn
ls | grep power
cat /etc/os-release
less /etc/apt/sources.list
cd
wget https://packages.debian.org/jessie/amd64/pdns-recursor/download
rm download
wget http://ftp.it.debian.org/debian/pool/main/p/pdns-recursor/pdns-recursor_3.6.2-2+deb8u4_amd64.deb
dpkg -i pdns-recursor_3.6.2-2+deb8u4_amd64.deb
apt install liblua5.2-0
dpkg -i pdns-recursor_3.6.2-2+deb8u4_amd64.deb
wget https://dl.pdnsmanager.org/pdnsmanager-2.1.0.tar.gz
mkdir pdns
mv pdnsmanager-2.1.0.tar.gz pdns
cd pdns/
tar xzf pdnsmanager-2.1.0.tar.gz
ls
cd pdnsmanager-2.1.0/
ls
cd ..
mv pdnsmanager-2.1.0 powerdns-manager
mv powerdns-manager/ /var/www/
cd /var/www/
ls
cd /etc/apache2/
ls
cd ..
cd nginx/
ls
cd sites-available/
ls
dpkg -l | grep php
cd
cd /etc/nginx/sites-available/
ls
nano pdns.conf
less sistemainoperativo.conf
rm pdns.conf
cp sistemainoperativo.conf pdns.conf
nano pdns.conf
vi pdns.conf
nano pdns.conf
cd ..
ls
cd modules-enabled/
ls
cd ..
less fastcgi_params
less fastcgi.conf
cd /etc/letsencrypt/live/
ls
cd servizi.linux.it
ls
ls -lah
cd ..
cd servizi.linux.it-0001/
ls -lah
cd ..
less ../../nginx/sites-available/pdns.conf
cp servizi.linux.it-0001/ powerdns.servizi.linux.it
cp -r servizi.linux.it-0001/ powerdns.servizi.linux.it
curl ifconfig.me
ln -s /etc/nginx/sites-available/pdns.conf /etc/nginx/sites-enabled/
systemctl reload nginx
netstat -an
netstat -ant
mysql
cd /var/log/nginx/
ls
cat *powerdns*
cd /var/run/php
ls
nano /etc/nginx/sites-available/pdns.conf
systemctl reload nginx
mysql
cd /var/www/powerdns-manager/
ls
cd backend/
ls
cd con
cd config
ls
cd /var/run/php
cd /var/log/nginx/
cat *powerdns*
echo > *powerdns*
echo > powerdns.access.log
echo > powerdns.error.log
cat *powerdns*
less /etc/nginx/sites-available/pdns.conf
nginx -t /etc/nginx/sites-available/pdns.conf
nginx -T /etc/nginx/sites-available/pdns.conf
nginx -t
less /etc/nginx/sites-available/pdns.conf
less /etc/nginx/fastcgi_params
cd /var/www/
ls -lah
cd Servizi/
ls -lah
cd homepage/
ls
ls -lah
cd ..
ls
ls -lah
chmod -R 755 powerdns-manager/
chown -R root:roow powerdns-manager/
chown -R root:root powerdns-manager/
ls -lah
tail -f /var/log/nginx/powerdns.error.log
less /etc/nginx/sites-enabled/pdns.conf
ls -lah /var/run/php/
less /etc/nginx/sites-enabled/pdns.conf
nano /etc/nginx/sites-available/pdns.conf
systemctl reload nginx
tail -f /var/log/nginx/powerdns.error.log
systemctl
less /etc/nginx/sites-enabled/pdns.conf
nano /etc/nginx/sites-enabled/pdns.conf
vi /etc/nginx/sites-enabled/pdns.conf
systemctl reload nginx
vi /etc/nginx/sites-enabled/pdns.conf
systemctl reload nginx
tail -f /var/log/nginx/powerdns.error.log
ls
rm -rf powerdns-manager/
cd ..
cd /
ls
cd
ls
rm -rf pdns
rm pdns-recursor_3.6.2-2+deb8u4_amd64.deb
unlink /etc/nginx/sites-enabled/pdns.conf
rm /etc/nginx/sites-available/pdns.conf
rm /var/log/nginx/powerdns.*
ls
cd /etc/powerdns/
ls
nano pdns.conf
nano pdns.d/pdns.local.gmysql.conf
nano recursor.conf
systemctl start pdns-recursor
systemctl start pdn
systemctl start pdns
journalctl -xe
apt-cache search gmysql
ls -al /usr/lib/powerdns
less /usr/share/pdns-server/pdns.conf
cd /usr/lib/
ls
cd x86_64-linux-gnu/
ls
cd pdns/
ls
cd
cd /etc/pdns
cd /etc/
cd powerdns/
ls
nano recursor.conf
nano pdns.
nano pdns.conf
systemctl start pdns
journalctl -xe
ls /usr/lib/x86_64-linux-gnu/pdns/
cd
mkdir pdns
cd pdns/wget https://downloads.powerdns.com/releases/pdns-4.4.1.tar.bz2
cd pdns/
wget https://downloads.powerdns.com/releases/pdns-4.4.1.tar.bz2
tar xf pdns-4.4.1.tar.bz2
ls
cd pdns-4.4.1/
ls
./configure
make
ls
find . | grep si
find . | grep so
find . | grep gmy
apt install libedit-dev liblua5.1-0-dev
./configure
./configure --with-modules="bind gmysql" --without-lua
nano config
nano configure
apt-get install libboost-dev libboost-filesystem-dev libboost-serialization-dev libboost-system-dev libboost-thread-dev libboost-context-dev libboost-test-dev libssl-dev libboost-test-dev g++ make pkg-config libluajit-5.1-dev
apt update
apt-get install libboost-dev libboost-filesystem-dev libboost-serialization-dev libboost-system-dev libboost-thread-dev libboost-context-dev libboost-test-dev libboost-test-dev g++ make pkg-config libluajit-5.1-dev
./configure --with-modules="bind gmysql"
apt-cache search libcrypto
apt ubstakk libcrypto++-dev
apt install libcrypto++-dev
./configure --with-modules="bind gmysql"
apt update
wget https://packages.sury.org/php/apt.gpg
cat apt.gpg | apt-key add
wget http://download.jitsi.org/stable/Release.gpg
cat Release.gpg | apt-key add
apt update
nano /etc/apt/sources.list.d/jitsi-stable.list
mv /etc/apt/sources.list.d/jitsi-stable.list /etc/apt/
apt update
apt install libssl-dev
apt install libssl-dev libssl1.1
cd /usr/lib/x86_64-linux-gnu/pdns/
ls
wget http://89.40.142.156/pdns/libgmysqlbackend.so
systemctl start pdns
journalctl -ex
journalctl -xe
ls /usr/lib/x86_64-linux-gnu/pdns/libgmysqlbackend.so
ls /usr/lib/x86_64-linux-gnu/pdns/libgmysqlbackend.s
ls /usr/lib/x86_64-linux-gnu/pdns/libgmysqlbackend.so
ls-lah
ls -lah
systemctl stop pdns
systemctl start pdns
journalctl -xe
apt-get install libmysqlclient20
apt-cache search libmysql
cd ..
wget http://89.40.142.156/pdns/libmysqlclient.so.20.3.16
wget http://89.40.142.156/pdns/libmysqlclient.so.20
systemctl start pdns
journalctl -xe
apt-cache search libc
journalctl -xe
wget http://89.40.142.156/pdns/libc.so.6
systemctl start pdns
journalctl -xe
mv libc.so.6 /lib/x86_64-linux-gnu/
systemctl start pdns
journalctl -xe
rm /lib/x86_64-linux-gnu/libc.so.6
journalctl -xe
rm /lib/x86_64-linux-gnu/libc.so.6
apt
exit

Event Timeline

Session analysis

The above is a real user session retrieved from ~/.bash_history from a production Debian GNU/Linux stretch and below you can see it again, analyzed with some inline comments.

The user purpose was: installing PowerDNS on a production server.

NOTE: This session happened in real life, almost destroying an operating system. My comments are here to explain why this session should never happen again on a computer. This should NEVER happen on a laptop and should NEVER NEVER NEVER happen in any production environment. This analysis was done in order to help people understand the job of a system administrator and the risks of applying certain actions without the needed background or needed discussions.
$ cd

As soon as you log in you are already in your user home. So this command was useless.

$ apt install pdns

This package is not available in Debian stretch.

$ apt install powerdns powerdns-recursor powerdns-backend-mysql
$ apt-cache search powerdns

Note that also the package 'powerdns-recursor' does not exist in Debian stretch. So the user here is probably getting an error. So the user is trying to mix arguments to understand the issue.

$ apt install pdns-server pdns-tools pdns-backend-mydns pdns-recursor

Read above.

$ apt install pdns-server pdns-tools pdns-backend-mydns

Now the user was able to install some packages and discovered that the problem was with the 'pdns-recursor' package that, as already said, is not available in Debian stretch.

IMPORTANT: IF YOU ARE IN THIS SITUATION where a package is not available for your distribution in a PRODUCTION server, STOP and TALK with your coworkers to find the right solution.

IMPORTANT: AGAIN - AT THIS POINT STOP. In fact, the next interaction was FATAL.

$ journalctl -xe

Now. Probably the user is probably seeing an error about a missing 'recursor' component causing errors during its service startup.

$ apt-cache search pdns

The user is trying to understand which packages are available.
You do NOT see 'pdns-recursor' so the next command has no sense.

$ apt-get install pdns-recursor

The above command has no sense since the user examined the cache.

$ cd /etc/
$ ls | grep pdn
$ ls | grep power

The user is trying to detect if the package was automagically provided.

$ cat /etc/os-release
PRETTY_NAME="Debian GNU/Linux 9 (stretch)"
NAME="Debian GNU/Linux"
VERSION_ID="9"
VERSION="9 (stretch)"
ID=debian
HOME_URL="https://www.debian.org/"
SUPPORT_URL="https://www.debian.org/support"
BUG_REPORT_URL="https://bugs.debian.org/"


I've reported above what the user was seeing.
That's VERY important:
Note that this is a Debian STRETCH. And NOT jessie.
It's 100% clear from the above output. 


$ less /etc/apt/sources.list
deb http://deb.debian.org/debian stretch ...
...


Now. I've reported again what the user was seeing.
That's VERY important.
Here you are 100% sure that this is a Debian STRETCH.

$ cd
$ wget https://packages.debian.org/jessie/amd64/pdns-recursor/download

Now the user discovered that the package was not available in STRETCH and so tried to download it from JESSIE.
THIS IS VERY IMPORTANT - DO NOT DO THIS AT HOME - DO NOT DO THIS ON YOUR LAPTOP - DO NOT NEVER NEVER NEVER NEVER DO THIS ON A PRODUCTION SERVER - YOU ARE DESTROYING YOUR DISTRIBUTION
https://wiki.debian.org/DontBreakDebian

As a side note, the user downloaded an HTML page and not a .deb package.

$ rm download 

The user probably tried to install 'download' and noticed that it's HTML and not a .deb.

$ wget http://ftp.it.debian.org/debian/pool/main/p/pdns-recursor/pdns-recursor_3.6.2-2+deb8u4_amd64.deb

Now the user is downloading the .deb but, again, this is for JESSIE and not for STRETCH so the user is compromising the distribution. This operation is like putting gasoline into a Panda 4x4.
PLEASE READ THIS NOTE
https://wiki.debian.org/DontBreakDebian


$ dpkg -i pdns-recursor_3.6.2-2+deb8u4_amd64.deb 

Now the user is compromising the stability of the distribution installing the JESSIE package in STRETCH.

Note that the user executed this installation command without any previous discussion. Tje user was just 100% sure that it was good to do this kind of stuff in a production server.

$ apt install liblua5.2-0
$ dpkg -i pdns-recursor_3.6.2-2+deb8u4_amd64.deb 

See above.

$ wget https://dl.pdnsmanager.org/pdnsmanager-2.1.0.tar.gz

Now the user is trying to avoid the package manager, trying to override it, installing a software from a website.

Note that Microsoft Windows works in this way. In Microsoft Windows an user can try to install whatever 'exe' malware from the Internet. That's not the real world. NEVER install a software from a website on a production server without any discussion with the team or your coworkers. This is not your laptop. NEVER wget-pipe-sudo in production without any discussion.

$ mkdir pdns
$ mv pdnsmanager-2.1.0.tar.gz pdns
$ cd pdns/
$ tar xzf pdnsmanager-2.1.0.tar.gz 
$ ls
$ cd pdnsmanager-2.1.0/
$ ls
$ cd ..
$ mv pdnsmanager-2.1.0 powerdns-manager
$ mv powerdns-manager/ /var/www/
$ cd /var/www/
$ ls
$ cd /etc/apache2/
$ ls
$ cd ..
$ cd nginx/
$ ls
$ cd sites-available/
$ ls

Read above.

$ dpkg -l | grep php
$ cd 
$ cd /etc/nginx/sites-available/
$ ls
$ nano pdns.conf
$ less sistemainoperativo.conf 
$ rm pdns.conf 
$ cp sistemainoperativo.conf pdns.conf
$ nano pdns.conf 
$ vi pdns.conf 
$ nano pdns.conf 
$ cd ..
$ ls
$ cd modules-enabled/
$ ls
$ cd ..
$ less fastcgi_params 
$ less fastcgi.conf 
$ cd /etc/letsencrypt/live/
$ ls
$ cd servizi.linux.it
$ ls
$ ls -lah
$ cd ..
$ cd servizi.linux.it-0001/
$ ls -lah
$ cd ..
$ less ../../nginx/sites-available/pdns.conf 
$ cp servizi.linux.it-0001/ powerdns.servizi.linux.it
$ cp -r servizi.linux.it-0001/ powerdns.servizi.linux.it
$ curl ifconfig.me

This is a public server and the IP can be analyzed using 'ip a' and looking at the eth0 interface.

$ ln -s /etc/nginx/sites-available/pdns.conf /etc/nginx/sites-enabled/

That's the right way to install a website in nginx in Debian. So the user is experienced with Debian. So it's difficult to explain why the previous operations were carried out.

$ systemctl reload nginx
$ netstat -an
$ netstat -ant
$ mysql
$ cd /var/log/nginx/
$ ls
$ cat *powerdns*
$ cd /var/run/php
$ ls
$ nano /etc/nginx/sites-available/pdns.conf 
$ systemctl reload nginx
$ mysql
$ cd /var/www/powerdns-manager/
$ ls
$ cd backend/
$ ls
$ cd con
$ cd config
$ ls
$ cd /var/run/php
$ cd /var/log/nginx/
$ cat *powerdns*
$ echo > *powerdns*

This command cannot work because the expansion generates 'echo > file1 file2 file3 file4' and this is not a valid syntax.

$ echo > powerdns.access.log 
$ echo > powerdns.error.log

Instead of clearing the logs you should just run 'tail -f file.log' to follow new lines.

$ cat *powerdns*
$ less /etc/nginx/sites-available/pdns.conf 
$ nginx -t /etc/nginx/sites-available/pdns.conf 
$ nginx -T /etc/nginx/sites-available/pdns.conf 
$ nginx -t
$ less /etc/nginx/sites-available/pdns.conf 
$ less /etc/nginx/fastcgi_params
$ cd /var/www/
$ ls -lah
$ cd Servizi/
$ ls -lah
$ cd homepage/
$ ls
$ ls -lah
$ cd ..
$ ls
$ ls -lah

?

$ chmod -R 755 powerdns-manager/
$ chown -R root:roow powerdns-manager/

Please look twice before issuing any command.

$ chown -R root:root powerdns-manager/
$ ls -lah
$ tail -f /var/log/nginx/powerdns.error.log 
$ less /etc/nginx/sites-enabled/pdns.conf 
$ ls -lah /var/run/php/
$ less /etc/nginx/sites-enabled/pdns.conf 
$ nano /etc/nginx/sites-available/pdns.conf 
$ systemctl reload nginx
$ tail -f /var/log/nginx/powerdns.error.log 
$ systemctl
$ less /etc/nginx/sites-enabled/pdns.conf 
$ nano /etc/nginx/sites-enabled/pdns.conf 
$ vi /etc/nginx/sites-enabled/pdns.conf 
$ systemctl reload nginx
$ vi /etc/nginx/sites-enabled/pdns.conf 
$ systemctl reload nginx
$ tail -f /var/log/nginx/powerdns.error.log 
$ ls
$ rm -rf powerdns-manager/
$ cd ..
$ cd /
$ ls
$ cd
$ ls
$ rm -rf pdns
$ rm pdns-recursor_3.6.2-2+deb8u4_amd64.deb 
$ unlink /etc/nginx/sites-enabled/pdns.conf 
$ rm /etc/nginx/sites-available/pdns.conf 
$ rm /var/log/nginx/powerdns.*
$ ls
$ cd /etc/powerdns/
$ ls
$ nano pdns.conf 
$ nano pdns.d/pdns.local.gmysql.conf
$ nano recursor.conf 
$ systemctl start pdns-recursor
$ systemctl start pdn
$ systemctl start pdns
$ journalctl -xe

Probably now the user is seeing some issues about wrong libraries.

$ apt-cache search gmysql
$ ls -al /usr/lib/powerdns
$ less /usr/share/pdns-server/pdns.conf


$ cd /usr/lib/
$ ls
$ cd x86_64-linux-gnu/

The user is examining a reserved directory that should not be touched if you do not want to destroy your distribution.

$ ls
$ cd pdns/
$ ls
$ cd 
$ cd /etc/pdns
$ cd /etc/
$ cd powerdns/
$ ls
$ nano recursor.conf 
$ nano pdns.
$ nano pdns.conf 
$ systemctl start pdns
$ journalctl -xe
$ ls /usr/lib/x86_64-linux-gnu/pdns/

Again the user is tempted to do bad things in this place.

$ cd
$ mkdir pdns
$ cd pdns/wget https://downloads.powerdns.com/releases/pdns-4.4.1.tar.bz2
$ cd pdns/
$ wget https://downloads.powerdns.com/releases/pdns-4.4.1.tar.bz2

Again the user is trying to install a software from the web on a production server without any discussion with anybody and without validating the fingerprint or anything.

$ tar xf pdns-4.4.1.tar.bz2 
$ ls
$ cd pdns-4.4.1/
$ ls
$ ./configure 
$ make

Probably it's showing missing dependencies.

$ ls
$ find . | grep si
$ find . | grep so
$ find . | grep gmy
$ apt install libedit-dev liblua5.1-0-dev 
$ ./configure 
$ ./configure --with-modules="bind gmysql" --without-lua 
$ nano config
$ nano configure
$ apt-get install libboost-dev libboost-filesystem-dev libboost-serialization-dev   libboost-system-dev libboost-thread-dev libboost-context-dev libboost-test-dev   libssl-dev libboost-test-dev g++ make pkg-config libluajit-5.1-dev
$ apt update
$ apt-get install libboost-dev libboost-filesystem-dev libboost-serialization-dev   libboost-system-dev libboost-thread-dev libboost-context-dev libboost-test-dev    libboost-test-dev g++ make pkg-config libluajit-5.1-dev
$ ./configure --with-modules="bind gmysql"
$ apt-cache search libcrypto
$ apt ubstakk libcrypto++-dev

Again please look twice before issuing any command.

$ apt install libcrypto++-dev
$ ./configure --with-modules="bind gmysql"
$ apt update

This is not needed twice. Maybe the user is following a copy-paste guides without thinking.

$ wget https://packages.sury.org/php/apt.gpg
$ cat apt.gpg | apt-key add

Now the user is again installing random keys from a website on a production server without any discussion with the team.

$ wget http://download.jitsi.org/stable/Release.gpg
$ cat Release.gpg | apt-key add

Read above. This command requires further investigation. This has not any explaination because Jitsi is unrelated from this changeset.

$ apt update

This is the third time the user runs this command. The user is following a copy-paste guides without thinking and is probably missing knowledge-base about Debian.

$ nano /etc/apt/sources.list.d/jitsi-stable.list 
$ mv /etc/apt/sources.list.d/jitsi-stable.list /etc/apt/

Why the user is installing Jitsi?

$ apt update

This is the fourth time this command has been run without changing anything.

$ apt install libssl-dev 
$ apt install libssl-dev libssl1.1

$ cd /usr/lib/x86_64-linux-gnu/pdns/
$ ls
$ wget http://89.40.142.156/pdns/libgmysqlbackend.so

VERY IMPORTANT NOTE
THE USER HERE HAS COMPROMISED THE SERVER

This is the public IP of a private server of the user and he is trying to download a library from his private server to this production server. Moreover the user declared that the server was Debian BUSTER (and not Debian STRETCH).
NEVER DO A SIMILAR THING
SO WE HAVE TWO ISSUES. ONE IS A SECURITY ISSUE and moreover the distribution is now compromised, installing a system library from an incompatible release.

PLEASE READ THIS NOTE
https://wiki.debian.org/DontBreakDebian

$ systemctl start pdns
$ journalctl -ex
$ journalctl -xe


$ ls /usr/lib/x86_64-linux-gnu/pdns/libgmysqlbackend.so
$ ls /usr/lib/x86_64-linux-gnu/pdns/libgmysqlbackend.s
$ ls /usr/lib/x86_64-linux-gnu/pdns/libgmysqlbackend.so
$ ls-lah
$ ls -lah

The user seems rushed and not very attentive but that's not a problem here.

$ systemctl stop pdns
$ systemctl start pdns
$ journalctl -xe
$ apt-get install libmysqlclient20
$ apt-cache search libmysql
$ cd ..



$ wget http://89.40.142.156/pdns/libmysqlclient.so.20.3.16
$ wget http://89.40.142.156/pdns/libmysqlclient.so.20


THE SERVER WAS COMPROMISED AGAIN BY THE USER
NEVER DO A SIMILAR THING
THIS IS A SECURITY ISSUE and moreover the user is installing INCOMPATIBLE binaries from incompatible distributions.

PLEASE READ THIS NOTE
https://wiki.debian.org/DontBreakDebian


$ systemctl start pdns
$ journalctl -xe
$ apt-cache search libc
$ journalctl -xe
$ wget http://89.40.142.156/pdns/libc.so.6

WHAT THE FUCK
THE libc.so.6 IS USED BY THE WHOLE OPERATING SYSTEM
NEVER REMOVE SYSTEM LIBRARIES ON A PRODUCTION SERVER
NEVER DO THIS ON YOUR LAPTOP AND NEVER NEVER NEVER NEVER DO THIS ON A PRODUCTION SERVER

$ systemctl start pdns
$ journalctl -xe
$ mv libc.so.6 /lib/x86_64-linux-gnu/

NOW THE SERVER IS TOTALLY COMPROMISED AND WILL GO IN SEGMENTATION FAULT SOON

$ systemctl start pdns
$ journalctl -xe
$ rm /lib/x86_64-linux-gnu/libc.so.6 

WHAT - THE - FUCK

$ journalctl -xe
$ rm /lib/x86_64-linux-gnu/libc.so.6 

AGAIN????? WHAT THE FUCK

$ apt

Probably the user was trying to use the auto-completion but the operating system is now in segmentation fault.

$ exit

Now nobody was able to enter via ssh and the system was failing to boot after a restart. We were not even able to chroot successfully without a segmentation fault.

I will try to add my session and how you can fix suck disaster.

IMPORTANT: Do not do it again. Always ask if you have any question on a production server and, if you have no questions, probably you have a problem.
valerio.bozzolan changed the title of this paste from How to destroy a Debian server to How to destroy a production Debian server.Jun 30 2021, 12:21