master_thesis/Chapters/Evaluation.tex

% !TeX root = ../Thesis.tex

%************************************************
\chapter{Evaluation}\label{ch:evaluation}
%************************************************
\glsresetall % Resets all acronyms to not used

% evaluation goal: use differential testing to analyze the security of esim on sim cards
% evaluation was performed on multiple esim on sim: in total 8 esims (shown in table)
% including the sysmoEUICC: special card -> loaded with gsma test certificate -> can also be used for smdp server with gsma test certificate -> testing of systems
% test ceritificate is freely available
% firmware implementation should not differ from normal esims
% altough for tests including an smdp server we have to use profiles with the test certificate
% estk.me does not offer an isd-r for regular interaction with our lpa implmenetation -> tesing is limited
% 9esim v2 offers both -> only testing isd-r not USAT for RSP related communication

% data fuzzing
% data fuzzing was performed on all esim except the estk.me esim since this card does not offer an isd-r 
% each fuzzing test was performed on all esims one after one


% findings

% estk.me fwupd
% estk.me offers an esim firmware update utility on their website
% this tool can be run on some host machine and comes with the newest firmware image of the esim
% the firmware image seems to be encrypted when looking at the shannon entropy (run this through shannon entropy tool and add actual entropy)
% the firmware update tool is a binary
% analysation with ghidra -> rev engeneering
% looking at dissassembled code -> trying to guess functions based on their implementation
% this is something unique to the estk.me esim since all other esims neither offer an public update utility nor having a custom publicly documented flash endpoint for newer firmware
% Etsi (check which version) defines that all sims and esim have to have some sort of update utility
% application updates can be perfomred via global platform commands although requiring the manufacturer keys to unlock the smartcard


% TODO: explain fwupd mechanism here
% update mechanism offers 2 functions: get_version and flash_firmware
% the custom flash endpoint is locate under the aid A06573746B6D65FFFFFFFF6677757064 for v1 and A06573746B6D65FFFFFFFFFFFF6D6774 for v2
% the card the i have for testing is an v1 version
% the aid is the hex encoded text 'estkmeÿÿÿÿfwupd' for v1 and 'estkmeÿÿÿÿÿÿmgt' for v2
% get_version retrieves the version code of the currently installed firmware by first selecting the aid and then sending the apdu packet APDU(class=0xAA, instruction=0xFF, p1=0x00, p2=0x00, le=0x08)
% our estk.me esim responds with Version 'T001V06' which corresponds to the 4.6.0 received via the euicc info 1
% this shows that this is an older estk.me model with the current one having the firmware version 'T3V7' as shown in newer firmware versions
% from the available firmware updates in the download page these versions exist: 
% T000V06-3.4.3
% T001V02-T001V05
% T001V03_V04-T001V05
% T001V05-T001V06
% T001V06-3.4.3
% T002V01-T002V06
% T002V06-3.4.3
% T003V03-4.2.4.1
% T003V03-4.4.3.1
% T003V03-4.5.1
% T003V03-T3V6SS0
% T3V4SS0-T3V6SS0
% T3V5SS0-T3V6SS0
% T3V6SS0-T3V7SS0 (v5.4.3)
% T001V06-3.4.3 seems to be the highest available version available for our card
% the list contains intermediate firmware versions which are used to upgrade from one major version to another i.e TXXXVXX-TXXVXX and current versions i.e TXXXVXX-X.X.X which are minor updates
% the list shows that there are 4 generations of cards: T000, T001, T002, and T003
% cards cannot be upgraded between generations
% only upugrade no downgrade support by the update mechanism -> autoselects the next firmware to update based on the installed one

% error handling is not really suported by the update mechanism -> wrong apdus or malformed apdus result in a CardConnectionException by the smartcard library which results from the pcsc-lite error SCARD_E_NOT_TRANSACTED (0x80100016)
% SCARD_E_NOT_TRANSACTED is thrown when the library tries to close a non existent connection \cite{https://pcsclite.apdu.fr/api/group__ErrorCodes.html} -> connection is already closed by the euicc -> euicc just closes the connection when erroring

% updating firmware
% the firmware update happens ins 6 Steps: setup, unlock, send_program_block, validate, and check_flash_status
% to initialise the fwup: select fwup endpoint and send APDU (cla=0x01, ins=0x55, p1=0x55, p2=0x55) -> init complete
% from here on the CLA for v0 APDUs is '00' and 'aa' for v1
% unlock the euicc: send apdu 'AA21000000'
% firmware is send in multiple blocks with the size 0x80D -> amount of blocks is defined by the firmware size i.e num_blocks = ceil(firmware_size / 0x80D)
% the mss is set to 255 bytes -> limit of normal APDU size defined by standard if not using extended APDUs (limit is 65536)
% each firmware block is sent in smaller APDUs where P1 indicates the current chunk of the program block and P2 the current_chunk & 0xFF
% after each program block is sent, the same data is sent again to euicc but the data part of the APDU is empty, the Le part of the APDU contains the chunk size and the instruction byte is set to 0x12 instead of 0x11 i.e go through each sent apdu and check that the received data length matches the chunk size provided in the Le of the APDU
% the update mechanism then sends the "AA13000000" APDU to check the flash status and indicate the next program block
% when all program blocks are sent -> send APDU 'AA00000000' to indicate the end of the flashing process
% using the information gathered through the reverse engeneering process -> reimplementation in python for further experimentation (also included in the resimulate as shown in implementation section TODO: cref to section)

% APDU mutation
% for APDU mutation we applied the same as shown in the mutation section i.e bitflip, random byte, zero block, shuffle block, truncate
% mutation are applied on a per chunk basis and in the same way when initially sending the program block
% same mutation is also applied when validating the blocks to ensure that the chunk size stays the same
% euicc accepts chunks initally until we send the check flash status APDU -> immediate crash
% indicates some sort of checksum or hash that is attached to each program block to ensure that the data received by the euicc is valid
% could not find any such thing in the binary -> road block thus could not find any further information about how this is beeing hashed or checksumed

% firmware binary analysis
% binary analysis was done with various tools like binwalk to check for known file types included in the firmware binary -> nothing
% entropy analysis with binwalk indicate that the firmware is encrypted (TODO: include image of entropy analysis)
% high shannon entropy of (TODO: check shannon entrpoy of all firmware versions)
% analysing the firmware files with ghidra also just showed just binary data without any structure



% tracing
% using simtrace 2 tracing and recording traffic showed that some esims use different aids for the isdr
% -> insert sample trace
% line X shows that the selected aid could not be mapped to anything known
% probably done by vendors to force people to use their own lpa implementations eventough this is also limited by the ara-m
% we found the following isdr aids 
% xesim: A0000005591010FFFFFFFF8900000177
% 5ber: A0000005591010FFFFFFFF8900050500
% esim.me: A0000005591010000000008900000300
% esim.me offers both their custom aid and the common aid to select the isd-r

% replaying the traffic to other esims did not show any relevant or interisting behaviour
% limited to small subset of esim functionality due to usage of nonces for specific tasks


% apdu fuzzing

% differences in execution time

% 

% error messages thrown by failed traces
% SCP03TSecurityError
% ApduException
% InvalidCertificate
% InvalidSignature
% UnsupportedRemoteOpType


% analyzing recorded apdu fuzzing
% PATHS = glob.glob("recordings/*.resim")
% 
% for path in PATHS:
%     with open(path, "rb") as f:
%         try:
%             recorder: OperationRecorder = pickle.load(f)
%         except Exception as e:
%             print(f"Failed to load {path}: {e}")
%             continue
% 
%         print(f"Processing {path}")
% 
%     def get_successfully_mutated_data(tree: MutationTreeNode) -> bool:
%         if tree.leaf:
%             return tree.failure_reason is None
% 
%         has_successful_child = False
% 
%         for child in tree.children:
%             is_successful = get_successfully_mutated_data(child)
% 
%             if is_successful:
%                 has_successful_child = True
% 
%             if (
%                 is_successful
%                 and tree.recording
%                 and tree.mutation_type != MutationType.NONE
%             ):
%                 diff = difflib.ndiff(
%                     tree.recording.original_apdu.data.hex(),
%                     tree.recording.mutated_apdu.data.hex(),
%                 )
%                 pprint(
%                     {
%                         "recording": path,
%                         "function": tree.func_name,
%                         "mutation_type": tree.mutation_type,
%                         "original_apdu": tree.recording.original_apdu.data.hex(),
%                         "mutated_apdu": tree.recording.mutated_apdu.data.hex(),
%                         "diff": "".join(diff),
%                     }
%                 )
% 
%         return has_successful_child
% 
%     get_successfully_mutated_data(recorder.root)
% 
% load recordings
% loop through nodes and find nodes where the branch is successfull and the mutation type is not none
% shows all successful mutations in the recodings folder
% ndiff: compares two strings and returns a diff view that shows the differences for both strings
% 

% get_euicc_info_1 truncate success
% truncation of trailing 00
% bf2000 -> bf20

% firstSequenceOf87 bitflip success
% diff:  a- 0+ 1  1  a  8  7  1  8  8  b  0  c  9  1  6  5  4  1  1  3  9  '
%         'd  f  b  1  d  4  f  0  6  3  c  7  b  c  2  d  7  4  b  d  f  1  2  '
%         'e  b  0  7  8  0  4  5  e  0  0  b'
% a0 -> a1
% indicates another channel was used


% authenticate_server was truncated but still successful
% last 75% of authenticate_server apdu are truncated
% only able to trigger 1 time -> problem: next mutation type selection was based on enum order
% solution: check if mutation in trace history was successful, if so: do not mutate anything and return MutationType.NONE; if no mutation happend: return a random mutation that was not yet used 
% from triggering it only once -> triggering it every run
% TODO: go into detail on what is happening exactly
% only happening on Eastcompeace euiccs


\begin{table}[ht]
    \centering
    \begin{tabular}{|l|l|l|l|l|l|l|c|}
        \hline
        \textbf{Name} & \textbf{Manufacturer} & \textbf{Firmware} & \textbf{SVN} & \textbf{Apps} \\
        \hline
        \hline
        9esim v2     & Kigen            & 36.18.5 & 2.3.0 & 12 \\
        9esim        & Kigen            & 36.17.4 & 2.3.0 & 11 \\
        sysmoEUICC   & Kigen            & 35.6.35 & 2.3.0 & 8  \\
        EIOTClub     & Kigen            & 36.7.24 & 2.3.0 & 10 \\
        esim.me      & Eastcompeace     & 4.2.0   & 2.2.0 & 0  \\
        estk.me      & G+D              & 4.6.0   & 2.2.0 & 0  \\
        5Ber         & Eastcompeace     & 4.2.0   & 2.2.2 & 0  \\
        Xesim        & Eastcompeace     & 4.2.0   & 2.2.2 & 0  \\
        \hline
    \end{tabular}
    \caption{Overview of Analyzed eSIM Cards}
    \label{tab:esim-overview}
\end{table}

\section{Design}

\section{Findings}
\label{sec:findings}

\section{estk Firmware Update Application}