Add sparse, improve logging.

Author: Bjoern Kerler

Config/qualcomm_config.py

@@ -408,6 +408,7 @@ class memory_type:
     nand = 0
     emmc = 1
     ufs = 2
+    spinor = 3
 
     preferred_memory = {
         "QDF2432": emmc,
@@ -473,12 +474,12 @@ class memory_type:
         "MDM9250": emmc,
         "MDM9350": emmc,
         "MDM9650": emmc,
-        "SDM429": ufs,
-        "SDM439": ufs,
-        "SDM450": ufs,
-        "SDM632": ufs,
-        "SDA632": ufs,
-        "SDX50M": ufs,
+        "SDM429": emmc,
+        "SDM439": emmc,
+        "SDM450": emmc,
+        "SDM632": emmc,
+        "SDA632": emmc,
+        "SDX50M": emmc,
         "qcs403": emmc,
         "qcs405": emmc,
         "ipq5018": emmc,
@@ -489,29 +490,29 @@ class memory_type:
         "MSM8930": emmc,
         "MSM8936": emmc,
 
-        "APQ8098": ufs,
+        "APQ8098": emmc,
         "MSM8998": ufs,
-        "SDM630": ufs,
-        "SDM636": ufs,
-        "SDM660": ufs,
-        "SDM670": ufs,
-        "SDA670": ufs,
-        "SDM710": ufs,
-        "QCS605": ufs,
+        "SDM630": emmc,
+        "SDM636": emmc,
+        "SDM660": emmc,
+        "SDM670": emmc,
+        "SDA670": emmc,
+        "SDM710": emmc,
+        "QCS605": emmc,
         "SXR1120": emmc,
         "SXR1130": emmc,
         "SDM845": ufs,
         "SDM850": ufs,
         "SDX24": emmc,
         "SDX24M": emmc,
-        "SDX55M": emmc,
-        "SDX55:CD90-PG591": emmc,
-        "SDX55:CD90-PH809": emmc,
-        "SA515M": ufs,
-        "SM6150": ufs,
-        "SM6150p": ufs,
-        "SM7150": ufs,
-        "SM7150p": ufs,
+        "SDX55M": ufs,
+        "SDX55:CD90-PG591": ufs,
+        "SDX55:CD90-PH809": ufs,
+        "SA515M": emmc,
+        "SM6150": emmc,
+        "SM6150p": emmc,
+        "SM7150": emmc,
+        "SM7150p": emmc,
         "SDM855_SM8150": ufs,
         "SDM855p_SM8150p": ufs,
         "SM8250": ufs,

Library/firehose.py

@@ -3,7 +3,7 @@
 import time
 from Library.utils import *
 from Library.gpt import gpt
-
+from Library.sparse import QCSparse
 
 try:
     from Library.Modules.init import modules
@@ -54,7 +54,7 @@ class cfg:
         MaxXMLSizeInBytes = 4096
         bit64 = True
 
-    def __init__(self, cdc, xml, cfg, log, devicemodel, serial, skipresponse, luns, args):
+    def __init__(self, cdc, xml, cfg, loglevel, devicemodel, serial, skipresponse, luns, args):
         self.cdc = cdc
         self.lasterror = b""
         self.args = args
@@ -67,8 +67,14 @@ def __init__(self, cdc, xml, cfg, log, devicemodel, serial, skipresponse, luns,
         self.skipresponse = skipresponse
         self.luns = luns
         self.supported_functions = []
-        if self.cfg.MemoryName == "UFS":
+        if self.cfg.MemoryName == "UFS" or self.cfg.MemoryName == "spinor":
             self.cfg.SECTOR_SIZE_IN_BYTES = 4096
+        self.__logger.setLevel(loglevel)
+        if loglevel==logging.DEBUG:
+            logfilename = "log.txt"
+            fh = logging.FileHandler(logfilename)
+            self.__logger.addHandler(fh)
+
 
     def detect_partition(self, arguments, partitionname):
         fpartitions = {}
@@ -275,7 +281,10 @@ def cmd_patch(self, physical_partition_number, start_sector, byte_offset, value,
     def cmd_program(self, physical_partition_number, start_sector, filename, display=True):
         size = os.stat(filename).st_size
         fsize = os.stat(filename).st_size
-        fname = os.path.basename(filename)
+        sparse = QCSparse(filename)
+        sparseformat = False
+        if sparse.readheader():
+            sparseformat = True
         with open(filename, "rb") as rf:
             # Make sure we fill data up to the sector size
             num_partition_sectors = size // self.cfg.SECTOR_SIZE_IN_BYTES
@@ -293,7 +302,6 @@ def cmd_program(self, physical_partition_number, start_sector, filename, display
             old = 0
             if num_partition_sectors<maxsectors:
                 maxsectors=num_partition_sectors
-
             for cursector in range(start_sector, start_sector + num_partition_sectors, maxsectors):
                 data = f"<?xml version=\"1.0\" ?><data>\n" + \
                        f"<program SECTOR_SIZE_IN_BYTES=\"{self.cfg.SECTOR_SIZE_IN_BYTES}\"" + \
@@ -313,7 +321,10 @@ def cmd_program(self, physical_partition_number, start_sector, filename, display
                         wlen = self.cfg.MaxPayloadSizeToTargetInBytes
                         if fsize < wlen:
                             wlen = fsize
-                        wdata = rf.read(wlen)
+                        if sparseformat:
+                            wdata=sparse.read(wlen)
+                        else:
+                            wdata = rf.read(wlen)
                         wlen=len(wdata)
                         bytesToWrite -= wlen
                         fsize -= wlen
@@ -740,15 +751,15 @@ def configure(self,lvl):
         self.__logger.info(f"TargetName={self.cfg.TargetName}")
         self.__logger.info(f"MemoryName={self.cfg.MemoryName}")
         self.__logger.info(f"Version={self.cfg.Version}")
-        if self.cfg.MemoryName.lower() == "emmc":
+        if self.cfg.MemoryName.lower() == "emmc" or self.cfg.MemoryName.lower()=="nand":
             self.cfg.SECTOR_SIZE_IN_BYTES = 512
-        elif self.cfg.MemoryName.lower() == "ufs":
+        elif self.cfg.MemoryName.lower() == "ufs" or self.cfg.MemoryName.lower() == "spinor":
             self.cfg.SECTOR_SIZE_IN_BYTES = 4096
 
         rsp=self.cmd_read_buffer(0,1,1,False)
         if rsp==-1:
                 if b"ERROR: Failed to initialize (open whole lun) UFS Device slot" in self.lasterror:
-                    self.__logger.warning("Memory type UFS doesn't seem to match (Failed to init). Use eMMC instead.")
+                    self.__logger.warning("Memory type UFS doesn't seem to match (Failed to init). Trying to use eMMC instead.")
                     self.cfg.MemoryName="eMMC"
                     return self.configure(0)
 

Library/firehose_client.py

@@ -65,6 +65,8 @@ def __init__(self, arguments, cdc, sahara, loglevel, printer):
                             type=memory_type.preferred_memory[self.target_name]
                             if type==memory_type.nand:
                                 self.cfg.MemoryName = "nand"
+                            if type==memory_type.spinor:
+                                self.cfg.MemoryName = "spinor"
                             elif type==memory_type.emmc:
                                 self.cfg.MemoryName = "eMMC"
                             elif type==memory_type.ufs:

Library/gpt.py

@@ -1,3 +1,4 @@
+#!/usr/bin/env python3
 import binascii
 from Library.utils import *
 
@@ -237,3 +238,23 @@ def generate_rawprogram(self, lun, sectorsize, directory):
             mstr += "</data>"
             wf.write(bytes(mstr, 'utf-8'))
             print(f"Wrote partition xml as {fname}")
+
+    def print_gptfile(self,filename):
+        with open(filename, "rb") as rf:
+            sectorsize = 4096
+            result=self.parse(rf.read(), sectorsize)
+            if result:
+                print(self.tostring())
+            return result
+        return False
+
+    def test_gpt(self):
+        res=self.print_gptfile(os.path.join("TestFiles", "gpt_sm8180x.bin"))
+        assert res,"GPT Partition wasn't decoded properly"
+
+if __name__=="__main__":
+    if len(sys.argv)<2:
+        print("Usage: ./gpt.py <filename>")
+        sys.exit()
+    gp = gpt()
+    gp.print_gptfile(sys.argv[1])

Library/sahara.py

@@ -2,6 +2,7 @@
 import time
 import os
 import sys
+import logging
 from struct import unpack, pack
 from Library.utils import read_object, print_progress, rmrf, LogBase
 from Config.qualcomm_config import sochw, msmids, root_cert_hash
@@ -273,7 +274,7 @@ def get_error_desc(self, status):
         ('filename', '20s')
     ]
 
-    def __init__(self, cdc):
+    def __init__(self, cdc, loglevel):
         self.cdc = cdc
         self.init_loader_db()
         self.version=2.1
@@ -291,6 +292,12 @@ def __init__(self, cdc):
         self.model_id = None
         self.oem_str = None
         self.msm_str = None
+        self.__logger.setLevel(loglevel)
+        if loglevel==logging.DEBUG:
+            logfilename = "log.txt"
+            fh = logging.FileHandler(logfilename)
+            self.__logger.addHandler(fh)
+
 
     def get_rsp(self):
         data = []
@@ -454,8 +461,8 @@ def info(self):
             self.serials = "{:08x}".format(self.serial)
             self.hwid = self.cmdexec_get_msm_hwid()
             self.pkhash = self.cmdexec_get_pkhash()
-            if self.version>=2.4:
-                self.sblversion = "{:08x}".format(self.cmdexec_get_sbl_version())
+            #if self.version>=2.4:
+            #    self.sblversion = "{:08x}".format(self.cmdexec_get_sbl_version())
             if self.hwid is not None:
                 self.hwidstr = "{:016x}".format(self.hwid)
                 self.msm_id = int(self.hwidstr[2:8], 16)
@@ -464,6 +471,7 @@ def info(self):
                 self.oem_str = "{:04x}".format(self.oem_id)
                 self.model_id = "{:04x}".format(self.model_id)
                 self.msm_str = "{:08x}".format(self.msm_id)
+                """
                 if self.version >= 2.4:
                     self.__logger.info(f"\n------------------------\n" +
                                 f"HWID:              0x{self.hwidstr} (MSM_ID:0x{self.msm_str}," +
@@ -473,7 +481,8 @@ def info(self):
                                 f"Serial:            0x{self.serials}\n" +
                                 f"SBL Version:       0x{self.sblversion}\n")
                 else:
-                    self.__logger.info(f"\n------------------------\n" +
+                """
+                self.__logger.info(f"\n------------------------\n" +
                                 f"HWID:              0x{self.hwidstr} (MSM_ID:0x{self.msm_str}," +
                                 f"OEM_ID:0x{self.oem_str}," +
                                 f"MODEL_ID:0x{self.model_id})\n" +
@@ -702,7 +711,7 @@ def upload_loader(self):
         if self.programmer == "":
             return ""
         try:
-            self.__logger.info(f"Using loader {self.programmer} ...")
+            self.__logger.info(f"Uploading loader {self.programmer} ...")
             with open(self.programmer, "rb") as rf:
                 programmer = rf.read()
         except Exception as e:

Library/sparse.py

@@ -0,0 +1,125 @@
+#!/usr/bin/env python3
+import sys
+from struct import unpack
+from queue import Queue
+try:
+    from Library.utils import LogBase, print_progress
+except Exception as e:
+    import os,sys,inspect
+    current_dir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))
+    parent_dir = os.path.dirname(current_dir)
+    sys.path.insert(0, parent_dir)
+    from Library.utils import LogBase, print_progress
+
+
+class QCSparse(metaclass=LogBase):
+    def __init__(self,filename):
+        self.rf=open(filename,'rb')
+        self.data=Queue()
+        self.offset=0
+        self.tmpdata=bytearray()
+
+    def readheader(self):
+        header = unpack("<I4H4I", self.rf.read(0x1C))
+        magic = header[0]
+        self.major_version = header[1]
+        self.minor_version = header[2]
+        self.file_hdr_sz = header[3]
+        self.chunk_hdr_sz = header[4]
+        self.blk_sz = header[5]
+        self.total_blks = header[6]
+        self.total_chunks = header[7]
+        self.image_checksum = header[8]
+        if magic!=0xED26FF3A:
+            return False
+        if self.file_hdr_sz != 28:
+            self.__logger.error("The file header size was expected to be 28, but is %u." % self.file_hdr_sz)
+            return False
+        if self.chunk_hdr_sz != 12:
+            self.__logger.error("The chunk header size was expected to be 12, but is %u." % self.chunk_hdr_sz)
+            return False
+        return True
+
+    def unsparse(self):
+        if self.total_blks < self.offset:
+            self.__logger.error("The header said we should have %u output blocks, but we saw %u" % (self.total_blks, self.offset))
+            return -1
+        header = unpack("<2H2I", self.rf.read(self.chunk_hdr_sz))
+        chunk_type = header[0]
+        chunk_sz = header[2]
+        total_sz = header[3]
+        data_sz = total_sz - 12
+        if chunk_type == 0xCAC1:
+            if data_sz != (chunk_sz * self.blk_sz):
+                self.__logger.error("Raw chunk input size (%u) does not match output size (%u)" % (data_sz, chunk_sz * self.blk_sz))
+                return -1
+            else:
+                self.__logger.debug("Raw data")
+                data = self.rf.read(chunk_sz * self.blk_sz)
+                self.offset += chunk_sz
+                return data
+        elif chunk_type == 0xCAC2:
+            if data_sz != 4:
+                self.__logger.error("Fill chunk should have 4 bytes of fill, but this has %u" % data_sz)
+                return -1
+            else:
+                fill_bin = self.rf.read(4)
+                fill = unpack("<I", fill_bin)
+                self.__logger.debug(format("Fill with 0x%08X" % fill))
+                data = fill_bin * (chunk_sz * self.blk_sz // 4)
+                self.offset += chunk_sz
+                return data
+        elif chunk_type == 0xCAC3:
+            data=b'\x00' * chunk_sz * self.blk_sz
+            self.offset += chunk_sz
+            return data
+        elif chunk_type == 0xCAC4:
+            if data_sz != 4:
+                self.__logger.error("CRC32 chunk should have 4 bytes of CRC, but this has %u" % data_sz)
+                return -1
+            else:
+                crc_bin = self.rf.read(4)
+                crc = unpack("<I", crc_bin)
+                self.__logger.debug(format("Unverified CRC32 0x%08X" % crc))
+                return b""
+        else:
+            self.__logger.debug("Unknown chunk type 0x%04X" % chunk_type)
+            return -1
+
+    def read(self,length=None):
+        if length==None:
+            return self.unsparse()
+        if length<=len(self.tmpdata):
+            tdata=self.tmpdata[:length]
+            self.tmpdata=self.tmpdata[length:]
+            return tdata
+        while len(self.tmpdata)<length:
+            self.tmpdata.extend(self.unsparse())
+            if length<=len(self.tmpdata):
+                tdata = self.tmpdata[:length]
+                self.tmpdata = self.tmpdata[length:]
+                return tdata
+
+if __name__=="__main__":
+    if len(sys.argv)<3:
+        print("./sparse.py <sparse_partition.img> <outfile>")
+        sys.exit()
+    sp=QCSparse(sys.argv[1])
+    if sp.readheader():
+        print("Extracting sectors to "+sys.argv[2])
+        with open(sys.argv[2],"wb") as wf:
+            old=0
+            while sp.offset<sp.total_blks:
+                prog=int(sp.offset / sp.total_blks * 100)
+                if prog>old:
+                    print_progress(prog, 100, prefix='Progress:', suffix='Complete', bar_length=50)
+                    old=prog
+                data=sp.read()
+                if data==b"" or data==-1:
+                    break
+                wf.write(data)
+            if len(sp.tmpdata)>0:
+                wf.write(sp.tmpdata)
+                sp.tmpdata=bytearray()
+            print_progress(100, 100, prefix='Progress:', suffix='Complete', bar_length=50)
+        print("Done.")

Library/usblib.py

@@ -58,6 +58,7 @@ def __init__(self, loglevel=logging.INFO, portconfig=None, devclass=-1):
             self.__logger.addHandler(fh)
 
     def verify_data(self, data, pre="RX:"):
+        self.__logger.debug("",stack_info=True)
         if isinstance(data, bytes) or isinstance(data, bytearray):
             if data[:5] == b"<?xml":
                 try: