begin to analyze server return message

client.py

@@ -27,6 +27,7 @@ import argparse
 import http.client
 import crypt
 import logging
+import os
 import os.path
 import sys
 import urllib.request
@@ -39,6 +40,7 @@ import random
 import lzma
 import socket
 import struct
+import xml.etree.ElementTree as ET
 
 
 class BitStream():
@@ -47,6 +49,15 @@ class BitStream():
         self._read = 0
         self._tampon = []
 
+    def needRead(self):
+        return self._pos - self._read
+
+    def sizeData(self):
+        return self._pos
+
+    def sizeRead(self):
+        return self._read
+
     # ------------------------------------
     def internalSerial(self, value, nbits):
         if nbits == 0:
@@ -126,7 +137,7 @@ class BitStream():
         self.internalSerial(v, 8)
 
     def pushString(self, valeur):
-        size=len(valeur)
+        #size=len(valeur)
         #self.internalSerial(size, 32)
         self.pushUint32(len(valeur))
         for x in valeur:
@@ -225,6 +236,11 @@ class BitStream():
             tmp += x
             _size -= 1
         return tmp
+    def readArrayChar(self, size):
+        ret = []
+        for i in range(0, size):
+            ret.append(self.readChar())
+        return ret
 
     # ------------------------------------
     def __str__(self):
@@ -242,6 +258,20 @@ class BitStream():
         self._tampon = [int(x) for x in data]
         self._pos = len(self._tampon) * 8
 
+    def showLastData(self):
+        ret = ""
+        readBefore = self._read
+        while self._read < self._pos:
+            if self._pos - self._read >= 8:
+                data = self.readUint8()
+            else:
+                data = self.readSerial(self._pos - self._read)
+            if ret != "":
+                ret += "."
+            ret += hex(data)
+        self._read = readBefore
+        return ret
+
 def Test():
     a = BitStream()
     a.pushBool(True)
@@ -300,6 +330,83 @@ def Test():
     print(b.readString())
     print(b.toBytes())
 
+class CFileChild():
+    def __init__(self, name, pos, size):
+        self.name = name
+        self.pos = pos
+        self.size = size
+
+    def __str__(self):
+        return self.name + '(pos:' + str(self.pos) + ', size:' + str(self.size) + ')'
+
+class CFileList():
+    def __init__(self, name, fullpath):
+        self.name = name
+        self.fullpath = fullpath
+        self.child = []
+
+    def addchild(self, name, pos, size):
+        child = CFileChild(name, pos, size)
+        self.child.append(child)
+
+    def __str__(self):
+        return self.name + '[' + ', '.join([str(x) for x in self.child]) + ']'
+
+class CFileContainer():
+    def __init__(self):
+        self.log = logging.getLogger('myLogger')
+        self.list = []
+
+    def addSearchPath(self, path):
+        if not path:
+            return
+        self.log.debug("read path:" + str(path))
+        onlyfiles = [f for f in os.listdir(path) if os.path.isfile(os.path.join(path, f))]
+        self.log.debug("read files:" + ','.join(onlyfiles))
+        for filename in onlyfiles:
+            extension = os.path.splitext(filename)[1]
+            if extension == '.bnp':
+                # Container for multi file
+                fullpath = os.path.join(path, filename)
+                size = os.path.getsize(fullpath)
+                data = CFileList(filename, fullpath)
+                with open(fullpath,  'rb') as fp:
+                    fp.seek(size-4)
+                    nOffsetFromBeginning = int.from_bytes(fp.read(4), byteorder='little', signed=False)
+                    self.log.debug("[%s] nOffsetFromBeginning:%u" % (filename, nOffsetFromBeginning))
+                    fp.seek(nOffsetFromBeginning)
+                    nNbFile = int.from_bytes(fp.read(4), byteorder='little', signed=False)
+                    self.log.debug("[%s] nNbFile:%u" % (filename, nNbFile))
+                    for i in range(0, nNbFile):
+                        nStringSize = int.from_bytes(fp.read(1), byteorder='little', signed=False)
+                        FileName = fp.read(nStringSize).decode()
+                        nFileSize2 = int.from_bytes(fp.read(4), byteorder='little', signed=False)
+
+                        nFilePos = int.from_bytes(fp.read(4), byteorder='little', signed=False)
+                        self.log.debug("[%s] (%d) sizestring:%d file:%s size2:%d pos:%d" % (filename, i, nStringSize, FileName, nFileSize2, nFilePos))
+                        data.addchild(FileName, nFilePos, nFileSize2)
+                    fp.close()
+                self.list.append(data)
+
+    def search(self, name):
+        for x in self.list:
+            for y in x.child:
+                if y.name == name:
+                    self.log.debug("file:%s child:%s pos:%d size:%d", x.name, y.name, y.pos, y.size)
+                    return  x.fullpath, y.pos, y.size
+        self.log.debug('-'*80)
+        return None, None, None
+
+    def getdata(self, name):
+        fullpath, pos, size = self.search(name)
+        self.log.debug("file:%s pos:%d size:%d", fullpath, pos, size)
+        data = None
+        with open(fullpath,  'rb') as fp:
+            fp.seek(pos)
+            data = fp.read(size)
+            fp.close()
+        return data
+
 
 class TConnectionState(IntEnum):
     NotInitialised = 0      # nothing happened yet
@@ -380,10 +487,10 @@ class CBNPCategorySet:
         self._IsIncremental = False
         self._CatRequired = ""
         self._Hidden = False
-        self._Files = ""
+        self._Files = []
 
     def __str__(self):
-        return self._Name + ' (IsOptional:' + str(self._IsOptional) + ', UnpackTo:' + self._UnpackTo + ', IsIncremental:' + str(self._IsIncremental) + ', CatRequired:' + self._CatRequired + ', Hidden:' + str(self._Hidden) + ', Files:' + self._Files + ')'
+        return self._Name + ' (IsOptional:' + str(self._IsOptional) + ', UnpackTo:' + self._UnpackTo + ', IsIncremental:' + str(self._IsIncremental) + ', CatRequired:' + self._CatRequired + ', Hidden:' + str(self._Hidden) + ', Files:' + str(self._Files) + ')'
 
 # #####################################################
 # persistent_data.h:140  #  struct CArg
@@ -1332,7 +1439,7 @@ class CPersistentDataRecord:
                 continue
             elif nextToken == __Tok_Files:
                 self.log.debug("__Tok_Files")
-                _CBNPCategory._Files = self.popString(nextToken)
+                _CBNPCategory._Files.append(self.popString(nextToken))
                 self.log.debug("_Files: %s" % str(_CBNPCategory._Files))
                 continue
             # Vidage des autres clefs (inconnues)
@@ -1353,25 +1460,24 @@ class CPersistentDataRecord:
 
 class ClientNetworkConnection:
     def __init__(self,
-                 khanaturl,
+                 khanat_host,
+                 khanat_port_frontend,
                  LanguageCode="fr"):
         self.log = logging.getLogger('myLogger')
         self._CurrentSendNumber = 0
         self.LanguageCode = LanguageCode
         self._QuitId = 0
-        self._ConnectionState = TConnectionState
+        self._ConnectionState = TConnectionState.NotInitialised
         self.UserAddr, self.UserKey, self.UserId = None, None, None
-        self.khanaturl = khanaturl
+        self.frontend = (khanat_host, khanat_port_frontend)
-        _khanaturl =  self.khanaturl.strip('"').strip("'")
-        try:
-            host, port = _khanaturl.split(':')
-        except:
-            host = _khanaturl
-            port = 47851
-            (1024).to_bytes(2, byteorder='big')
-        self.frontend = (host, port)
         self._sock = socket.socket(socket.AF_INET, # Internet
                             socket.SOCK_DGRAM) # UDP
+        self._CurrentReceivedNumber = 0
+        self._SystemMode = 0
+        self._LastReceivedAck = 0
+        self._LatestProbe = 0
+        self._LastReceivedNumber = 0
+        self._LastAckInLongAck = 0
 
     def cookiesInit(self, UserAddr, UserKey, UserId):
         self.UserAddr = UserAddr
@@ -1413,6 +1519,18 @@ class ClientNetworkConnection:
         self._sock.sendto(msg.toBytes(), self.frontend)
         self._ConnectionState = TConnectionState.Quit
 
+    def sendSystemAckSync(self): # code/ryzom/client/src/network_connection.cpp # void  CNetworkConnection::sendSystemAckSync()
+        if self._sock is None:
+            raise ValueError
+        msg = BitStream()
+        self.buildSystemHeader(msg)
+        msg.pushUint8(2) # SYSTEM_ACK_SYNC_CODE
+        msg.pushSint32(self._LastReceivedNumber)
+        msg.pushSint32(self._LastAckInLongAck)
+        # msg.pushSint32(self._LongAckBitField) #
+        # msg.pushSint32(self._LatestSync)
+        self.log.error("TODO")
+
     def readDelta(self, msg):
         propertyCount = msg.readUint16()
         self.log.debug("propertyCount:%d" % propertyCount)
@@ -1420,22 +1538,82 @@ class ClientNetworkConnection:
             pass
 
     def impulseCallBack(self, data):
-        # code/ryzom/common/src/game_share/generic_xml_msg_mngr.h : CNode	*select(NLMISC::CBitMemStream &strm)
+        # code/ryzom/common/src/game_share/generic_xml_msg_mngr.h : CNode   *select(NLMISC::CBitMemStream &strm)
         msg = BitStream()
         msg.fromBytes(data)
         serverTick = msg.readUint32()
         self.log.debug("serverTick:%d" % serverTick)
         #self.readDelta(msg)
 
+    # khanat-opennel-code/code/ryzom/client/src/network_connection.cpp # bool CNetworkConnection::buildStream( CBitMemStream &msgin )
+    def buildStream(self, buffersize=65536):
+        data, addr = self._sock.recvfrom(buffersize)
+        return data, addr
+
+    def decodeHeader(self, msg):
+        self._CurrentReceivedNumber = msg.readSint32()
+        self._SystemMode = msg.readBool()
+        if self._SystemMode:
+            return
+        self._LastReceivedAck = msg.readSint32();
+        self._LastReceivedNumber = self._CurrentReceivedNumber
+
+    def receiveSystemProbe(self, msg):
+        self._LatestProbe = msg.readSint32()
+        self.log.debug("LatestProbe: %d" % self._LatestProbe)
+
+    def receiveSystemStalled(self, msg):
+        self.log.debug("received STALLED")
+
+    def receiveSystemSync(self, msg):
+        _Synchronize = msg.readUint32()
+        stime = msg.readSint64()
+        _LatestSync = msg.readUint32()
+        self.log.debug("%d %d %d" %(_Synchronize, stime, _LatestSync))
+        # khanat-opennel-code/code/ryzom/client/src/network_connection.cpp : void CNetworkConnection::receiveSystemSync(CBitMemStream &msgin)
+        MsgData = msg.readArrayChar(16)
+        DatabaseData = msg.readArrayChar(16)
+        self.log.debug("MsgData:" + str(MsgData))
+        self.log.debug("DatabaseData:" + str(DatabaseData))
+        self.log.error("TODO")
+        #self.sendSystemAckSync()
+
+    def disconnect(self):
+        pass
+
     def EmulateFirst(self):
         self.log.info("Client Login")
         self.sendSystemLogin()
 
         self.log.info("Receive Message")
         for _ in range(0, 20): # while True:
-            data, addr = self._sock.recvfrom(1024) # buffer size is 1024 bytes
+            data, addr = self.buildStream()
             self.log.debug("received message: %s" % data)
-            self.impulseCallBack(data)
+            msg = BitStream()
+            msg.fromBytes(data)
+            self.decodeHeader(msg)
+            # khanat-opennel-code/code/ryzom/client/src/network_connection.cpp:bool CNetworkConnection::stateSynchronize()
+            message = msg.readUint8()
+            self.log.debug("_CurrentReceivedNumber:%d (mode:%s) %d [%d/%d/%d]" % (self._CurrentReceivedNumber, str(self._SystemMode), message, msg.sizeData(),  msg.sizeRead(), msg.needRead()))
+            if message == 1: # SYSTEM_SYNC_CODE
+                self.log.debug("synchronize->synchronize")
+                self.receiveSystemSync(msg)
+            elif message == 3: # SYSTEM_PROBE_CODE
+                self.log.debug("synchronize->probe")
+                self._ConnectionState = TConnectionState.Probe
+                self.receiveSystemProbe(msg)
+            elif message == 6: # SYSTEM_STALLED_CODE
+                self.log.debug("received STALLED")
+                self._ConnectionState = TConnectionState.Stalled
+                self.receiveSystemStalled(msg)
+            elif message == 7: # SYSTEM_SERVER_DOWN_CODE
+                self.disconnect()
+                self.log.warning("BACK-END DOWN")
+            else:
+                self.log.warning("CNET: received system %d in state Synchronize" % message)
+            self.log.debug("_CurrentReceivedNumber:%d (mode:%s) %d [%d/%d/%d] '%s'" % (self._CurrentReceivedNumber, str(self._SystemMode), message, msg.sizeData(),  msg.sizeRead(), msg.needRead(), msg.showLastData()))
+
+            # self.impulseCallBack(data)
 
         self.log.info("Client Logout")
         self.sendSystemQuit()
@@ -1443,7 +1621,9 @@ class ClientNetworkConnection:
 
 class ClientKhanat:
     def __init__(self,
-                 khanaturl,
+                 khanat_host,
+                 khanat_port_login = 40916,
+                 khanat_port_frontend = 47851,
                  login="tester",
                  password="tester",
                  clientApp="Lirria",
@@ -1451,7 +1631,8 @@ class ClientKhanat:
                  url="/login/r2_login.php",
                  suffix = None,
                  download_patch = False,
-                 show_patch_detail=False):
+                 show_patch_detail=False,
+                 size_buffer_file=1024):
         self.log = logging.getLogger('myLogger')
 
         if suffix is None:
@@ -1460,7 +1641,9 @@ class ClientKhanat:
 
         self.download_patch = download_patch
         self.show_patch_detail = show_patch_detail
-        self.khanaturl = khanaturl
+        self.khanat_host = khanat_host
+        self.khanat_port_login = khanat_port_login
+        self.khanat_port_frontend = khanat_port_frontend
         self.login = login + suffix
         self.password = password
         self.clientApp = clientApp
@@ -1471,25 +1654,20 @@ class ClientKhanat:
         self.log.debug("Temporary directory:%s" % self.tempdir)
         self.khanat_idx = CPersistentDataRecord(self.log)
         self.UserAddr, self.UserKey, self.UserId = None, None, None
-        self.clientNetworkConnection = ClientNetworkConnection(khanaturl)
+        self.clientNetworkConnection = ClientNetworkConnection(self.khanat_host, self.khanat_port_frontend)
+        self.size_buffer_file = size_buffer_file
+        self.cFileContainer = CFileContainer()
 
     def createAccount(self):
-        khanaturl =  self.khanaturl.strip('"').strip("'")
+        conn = http.client.HTTPConnection(host=self.khanat_host, port=self.khanat_port_login)
-        try:
-            host, port = khanaturl.split(':')
-        except:
-            host = khanaturl
-            port = 40916
-
-        conn = http.client.HTTPConnection(host=host, port=port)
         cmd = "/ams/index.php?page=register"
         headers = {'Accept':  'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
                             'Accept-Language' : 'en-US',
                             'Connection': 'keep-alive',
                             'DNT': '1',
                             'Cookie': 'PHPSESSID=lsoumn9f0ljgm3vo3hgjdead03',
-                            'Host': khanaturl+':'+ str(port),
+                            'Host': self.khanat_host+':'+ str(self.khanat_port_login),
-                            'Referer': 'http://' + khanaturl+':'+ str(port) + '/ams/index.php?page=register',
+                            'Referer': 'http://' + self.khanat_host+':'+ str(self.khanat_port_login) + '/ams/index.php?page=register',
                             'Upgrade-Insecure-Requests': '1',
                             'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; rv:6.0) Gecko/20100101 Firefox/6.0',
                             'Content-Type': 'application/x-www-form-urlencoded'}
@@ -1533,14 +1711,7 @@ class ClientKhanat:
         self.log.info("Reuse account : %s" % self.login)
 
     def connectR2(self):
-        khanaturl =  self.khanaturl.strip('"').strip("'")
+        conn = http.client.HTTPConnection(host=self.khanat_host, port=self.khanat_port_login)
-        try:
-            host, port = khanaturl.split(':')
-        except:
-            host = khanaturl
-            port = 40916
-
-        conn = http.client.HTTPConnection(host=host, port=port)
         cmd = self.url + "?cmd=ask&cp=2&login=" + self.login + "&lg=" + self.LanguageCode
         conn.request("GET", cmd)
         response = conn.getresponse()
@@ -1563,7 +1734,7 @@ class ClientKhanat:
 
         cryptedPassword = crypt.crypt(self.password, salt)
 
-        conn = http.client.HTTPConnection(host=host, port=port)
+        conn = http.client.HTTPConnection(host=self.khanat_host, port=self.khanat_port_login)
         cmd = self.url + "?cmd=login&login=" + self.login + "&password=" + cryptedPassword + "&clientApplication=" + self.clientApp + "&cp=2" + "&lg=" + self.LanguageCode
         conn.request("GET", cmd)
         response = conn.getresponse()
@@ -1614,7 +1785,7 @@ class ClientKhanat:
                     break
             self.log.debug("size:%d", file_size)
             file_size_dl = 0
-            block_size = 1024  # 8192
+            block_size = self.size_buffer_file # 1024
 
             with open(dest, 'wb') as fp:
                 while True:
@@ -1651,6 +1822,65 @@ class ClientKhanat:
                     data = fin.read()
                     fout.write(data)
                 self.log.info("%s" % dstName)
+            os.remove(tmp)
+        # khanat-opennel-code/code/ryzom/client/src/login_patch.cpp # void CCheckThread::run ()
+        FilesToPatch = []
+        for file in self.khanat_idx.CBNPFile:
+            FilesToPatch.append(file)
+        # Here we got all the files to patch in FilesToPatch and all the versions that must be obtained Now we have to get the optional categories
+        OptionalCat = []
+        for category in self.khanat_idx.Categories:
+            if category._IsOptional:
+                for file in category._Files:
+                    bAdded = False
+                    for file2 in FilesToPatch:
+                        if file2 == file:
+                            OptionalCat.append(category._Name)
+                            bAdded = True
+                            break
+                    if bAdded:
+                        break
+        # For all categories that required an optional category if the cat required is present the category that reference it must be present
+        for category in self.khanat_idx.Categories:
+            if category._IsOptional and not len(category._CatRequired) == 0:
+                bFound = False
+                for cat in OptionalCat:
+                    if category._Name == cat:
+                        bFound = True
+                        break
+                if bFound:
+                    for cat in OptionalCat:
+                        if category._CatRequired == cat:
+                            OptionalCat.append(category._Name)
+                            break
+        # Delete categories optional cat that are hidden
+        for category in self.khanat_idx.Categories:
+            if category._IsOptional and category._Hidden:
+                for cat in OptionalCat:
+                    if category._Name == cat:
+                        OptionalCat.remove(category._Name)
+                        break
+        # Get all extract to category and check files inside the bnp with real files
+        for category in self.khanat_idx.Categories:
+            if len(category._UnpackTo) != 0:
+                for file in category._Files:
+                    # TODO
+                    # readHeader()
+                    pass
+
+    def DownloadMinimum(self):
+        self.log.debug("-" * 80)
+        for file in self.khanat_idx.CBNPFile:
+            if file.FileName != "kh_server.bnp":
+                continue
+            tmp = self.getServerFile("%05d/%s.lzma" % (int(self.r2serverversion), file.FileName), False, "")
+            with lzma.open(tmp) as fin:
+                dstName = os.path.join(self.tempdir.name, file.FileName)
+                with open(dstName, "wb") as fout:
+                    data = fin.read()
+                    fout.write(data)
+                self.log.info("%s" % dstName)
+            os.remove(tmp)
 
     def Emulate(self):
         self.createAccount()
@@ -1667,6 +1897,16 @@ class ClientKhanat:
         # Download all file in patch - login_patch.cpp:2578 # void CPatchThread::processFile (CPatchManager::SFileToPatch &rFTP)
         if self.download_patch:
             self.downloadAllPatch()
+        else:
+            self.DownloadMinimum()
+        self.cFileContainer = CFileContainer()
+        self.cFileContainer.addSearchPath(self.tempdir.name)
+        data = self.cFileContainer.getdata("msg.xml").decode()
+        msgXml = ET.fromstring(data)
+        ET.dump(msgXml)
+        data = self.cFileContainer.getdata("database.xml").decode()
+        databaseXml = ET.fromstring(data)
+        ET.dump(databaseXml)
         self.clientNetworkConnection.EmulateFirst()
 
 
@@ -1676,11 +1916,15 @@ def main():
     log = logging.getLogger('myLogger')
 
     parser = argparse.ArgumentParser()
-    parser.add_argument("--khanaturl", help="khanat URL to auhtenticate",  default='localhost')
+    parser.add_argument("--khanat-host", help="khanat host to auhtenticate",  default='localhost')
     parser.add_argument("--suffix", help="define suffix")
     parser.add_argument("-d", "--debug", help="show debug message", action='store_true')
     parser.add_argument("-p", "--download-patch", help="show debug message", action='store_true')
     parser.add_argument("-s", "--show-patch-detail", help="show debug message", action='store_true')
+    parser.add_argument("--size-buffer-file", help="size buffer to download file", type=int, default=1024)
+    parser.add_argument("--khanat-port-login", help="port http login", type=int, default=40916)
+    parser.add_argument("--khanat-port-frontend", help="port UDP frontend", type=int, default=47851)
+
     args = parser.parse_args()
 
     if args.debug:
@@ -1689,7 +1933,7 @@ def main():
         level = logging.getLevelName('INFO')
     log.setLevel(level)
 
-    client = ClientKhanat(args.khanaturl, suffix=args.suffix, download_patch=args.download_patch,  show_patch_detail=args.show_patch_detail)
+    client = ClientKhanat(args.khanat_host, khanat_port_login=args.khanat_port_login, khanat_port_frontend=args.khanat_port_frontend, suffix=args.suffix, download_patch=args.download_patch, show_patch_detail=args.show_patch_detail, size_buffer_file=args.size_buffer_file)
     client.Emulate()
     log.info("End")