From d3bab57963e13411b69dcf53b40276678e3f09f4 Mon Sep 17 00:00:00 2001 From: AleaJactaEst Date: Thu, 30 May 2019 19:19:54 +0200 Subject: [PATCH] begin to analyze server return message --- client.py | 332 ++++++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 288 insertions(+), 44 deletions(-) diff --git a/client.py b/client.py index a2491c4..8018149 100755 --- a/client.py +++ b/client.py @@ -27,6 +27,7 @@ import argparse import http.client import crypt import logging +import os import os.path import sys import urllib.request @@ -39,6 +40,7 @@ import random import lzma import socket import struct +import xml.etree.ElementTree as ET class BitStream(): @@ -47,6 +49,15 @@ class BitStream(): self._read = 0 self._tampon = [] + def needRead(self): + return self._pos - self._read + + def sizeData(self): + return self._pos + + def sizeRead(self): + return self._read + # ------------------------------------ def internalSerial(self, value, nbits): if nbits == 0: @@ -126,7 +137,7 @@ class BitStream(): self.internalSerial(v, 8) def pushString(self, valeur): - size=len(valeur) + #size=len(valeur) #self.internalSerial(size, 32) self.pushUint32(len(valeur)) for x in valeur: @@ -225,6 +236,11 @@ class BitStream(): tmp += x _size -= 1 return tmp + def readArrayChar(self, size): + ret = [] + for i in range(0, size): + ret.append(self.readChar()) + return ret # ------------------------------------ def __str__(self): @@ -242,6 +258,20 @@ class BitStream(): self._tampon = [int(x) for x in data] self._pos = len(self._tampon) * 8 + def showLastData(self): + ret = "" + readBefore = self._read + while self._read < self._pos: + if self._pos - self._read >= 8: + data = self.readUint8() + else: + data = self.readSerial(self._pos - self._read) + if ret != "": + ret += "." + ret += hex(data) + self._read = readBefore + return ret + def Test(): a = BitStream() a.pushBool(True) @@ -300,6 +330,83 @@ def Test(): print(b.readString()) print(b.toBytes()) +class CFileChild(): + def __init__(self, name, pos, size): + self.name = name + self.pos = pos + self.size = size + + def __str__(self): + return self.name + '(pos:' + str(self.pos) + ', size:' + str(self.size) + ')' + +class CFileList(): + def __init__(self, name, fullpath): + self.name = name + self.fullpath = fullpath + self.child = [] + + def addchild(self, name, pos, size): + child = CFileChild(name, pos, size) + self.child.append(child) + + def __str__(self): + return self.name + '[' + ', '.join([str(x) for x in self.child]) + ']' + +class CFileContainer(): + def __init__(self): + self.log = logging.getLogger('myLogger') + self.list = [] + + def addSearchPath(self, path): + if not path: + return + self.log.debug("read path:" + str(path)) + onlyfiles = [f for f in os.listdir(path) if os.path.isfile(os.path.join(path, f))] + self.log.debug("read files:" + ','.join(onlyfiles)) + for filename in onlyfiles: + extension = os.path.splitext(filename)[1] + if extension == '.bnp': + # Container for multi file + fullpath = os.path.join(path, filename) + size = os.path.getsize(fullpath) + data = CFileList(filename, fullpath) + with open(fullpath, 'rb') as fp: + fp.seek(size-4) + nOffsetFromBeginning = int.from_bytes(fp.read(4), byteorder='little', signed=False) + self.log.debug("[%s] nOffsetFromBeginning:%u" % (filename, nOffsetFromBeginning)) + fp.seek(nOffsetFromBeginning) + nNbFile = int.from_bytes(fp.read(4), byteorder='little', signed=False) + self.log.debug("[%s] nNbFile:%u" % (filename, nNbFile)) + for i in range(0, nNbFile): + nStringSize = int.from_bytes(fp.read(1), byteorder='little', signed=False) + FileName = fp.read(nStringSize).decode() + nFileSize2 = int.from_bytes(fp.read(4), byteorder='little', signed=False) + + nFilePos = int.from_bytes(fp.read(4), byteorder='little', signed=False) + self.log.debug("[%s] (%d) sizestring:%d file:%s size2:%d pos:%d" % (filename, i, nStringSize, FileName, nFileSize2, nFilePos)) + data.addchild(FileName, nFilePos, nFileSize2) + fp.close() + self.list.append(data) + + def search(self, name): + for x in self.list: + for y in x.child: + if y.name == name: + self.log.debug("file:%s child:%s pos:%d size:%d", x.name, y.name, y.pos, y.size) + return x.fullpath, y.pos, y.size + self.log.debug('-'*80) + return None, None, None + + def getdata(self, name): + fullpath, pos, size = self.search(name) + self.log.debug("file:%s pos:%d size:%d", fullpath, pos, size) + data = None + with open(fullpath, 'rb') as fp: + fp.seek(pos) + data = fp.read(size) + fp.close() + return data + class TConnectionState(IntEnum): NotInitialised = 0 # nothing happened yet @@ -380,10 +487,10 @@ class CBNPCategorySet: self._IsIncremental = False self._CatRequired = "" self._Hidden = False - self._Files = "" + self._Files = [] def __str__(self): - return self._Name + ' (IsOptional:' + str(self._IsOptional) + ', UnpackTo:' + self._UnpackTo + ', IsIncremental:' + str(self._IsIncremental) + ', CatRequired:' + self._CatRequired + ', Hidden:' + str(self._Hidden) + ', Files:' + self._Files + ')' + return self._Name + ' (IsOptional:' + str(self._IsOptional) + ', UnpackTo:' + self._UnpackTo + ', IsIncremental:' + str(self._IsIncremental) + ', CatRequired:' + self._CatRequired + ', Hidden:' + str(self._Hidden) + ', Files:' + str(self._Files) + ')' # ##################################################### # persistent_data.h:140 # struct CArg @@ -1332,7 +1439,7 @@ class CPersistentDataRecord: continue elif nextToken == __Tok_Files: self.log.debug("__Tok_Files") - _CBNPCategory._Files = self.popString(nextToken) + _CBNPCategory._Files.append(self.popString(nextToken)) self.log.debug("_Files: %s" % str(_CBNPCategory._Files)) continue # Vidage des autres clefs (inconnues) @@ -1353,25 +1460,24 @@ class CPersistentDataRecord: class ClientNetworkConnection: def __init__(self, - khanaturl, + khanat_host, + khanat_port_frontend, LanguageCode="fr"): self.log = logging.getLogger('myLogger') self._CurrentSendNumber = 0 self.LanguageCode = LanguageCode self._QuitId = 0 - self._ConnectionState = TConnectionState + self._ConnectionState = TConnectionState.NotInitialised self.UserAddr, self.UserKey, self.UserId = None, None, None - self.khanaturl = khanaturl - _khanaturl = self.khanaturl.strip('"').strip("'") - try: - host, port = _khanaturl.split(':') - except: - host = _khanaturl - port = 47851 - (1024).to_bytes(2, byteorder='big') - self.frontend = (host, port) + self.frontend = (khanat_host, khanat_port_frontend) self._sock = socket.socket(socket.AF_INET, # Internet socket.SOCK_DGRAM) # UDP + self._CurrentReceivedNumber = 0 + self._SystemMode = 0 + self._LastReceivedAck = 0 + self._LatestProbe = 0 + self._LastReceivedNumber = 0 + self._LastAckInLongAck = 0 def cookiesInit(self, UserAddr, UserKey, UserId): self.UserAddr = UserAddr @@ -1413,6 +1519,18 @@ class ClientNetworkConnection: self._sock.sendto(msg.toBytes(), self.frontend) self._ConnectionState = TConnectionState.Quit + def sendSystemAckSync(self): # code/ryzom/client/src/network_connection.cpp # void CNetworkConnection::sendSystemAckSync() + if self._sock is None: + raise ValueError + msg = BitStream() + self.buildSystemHeader(msg) + msg.pushUint8(2) # SYSTEM_ACK_SYNC_CODE + msg.pushSint32(self._LastReceivedNumber) + msg.pushSint32(self._LastAckInLongAck) + # msg.pushSint32(self._LongAckBitField) # + # msg.pushSint32(self._LatestSync) + self.log.error("TODO") + def readDelta(self, msg): propertyCount = msg.readUint16() self.log.debug("propertyCount:%d" % propertyCount) @@ -1420,22 +1538,82 @@ class ClientNetworkConnection: pass def impulseCallBack(self, data): - # code/ryzom/common/src/game_share/generic_xml_msg_mngr.h : CNode *select(NLMISC::CBitMemStream &strm) + # code/ryzom/common/src/game_share/generic_xml_msg_mngr.h : CNode *select(NLMISC::CBitMemStream &strm) msg = BitStream() msg.fromBytes(data) serverTick = msg.readUint32() self.log.debug("serverTick:%d" % serverTick) #self.readDelta(msg) + # khanat-opennel-code/code/ryzom/client/src/network_connection.cpp # bool CNetworkConnection::buildStream( CBitMemStream &msgin ) + def buildStream(self, buffersize=65536): + data, addr = self._sock.recvfrom(buffersize) + return data, addr + + def decodeHeader(self, msg): + self._CurrentReceivedNumber = msg.readSint32() + self._SystemMode = msg.readBool() + if self._SystemMode: + return + self._LastReceivedAck = msg.readSint32(); + self._LastReceivedNumber = self._CurrentReceivedNumber + + def receiveSystemProbe(self, msg): + self._LatestProbe = msg.readSint32() + self.log.debug("LatestProbe: %d" % self._LatestProbe) + + def receiveSystemStalled(self, msg): + self.log.debug("received STALLED") + + def receiveSystemSync(self, msg): + _Synchronize = msg.readUint32() + stime = msg.readSint64() + _LatestSync = msg.readUint32() + self.log.debug("%d %d %d" %(_Synchronize, stime, _LatestSync)) + # khanat-opennel-code/code/ryzom/client/src/network_connection.cpp : void CNetworkConnection::receiveSystemSync(CBitMemStream &msgin) + MsgData = msg.readArrayChar(16) + DatabaseData = msg.readArrayChar(16) + self.log.debug("MsgData:" + str(MsgData)) + self.log.debug("DatabaseData:" + str(DatabaseData)) + self.log.error("TODO") + #self.sendSystemAckSync() + + def disconnect(self): + pass + def EmulateFirst(self): self.log.info("Client Login") self.sendSystemLogin() self.log.info("Receive Message") for _ in range(0, 20): # while True: - data, addr = self._sock.recvfrom(1024) # buffer size is 1024 bytes + data, addr = self.buildStream() self.log.debug("received message: %s" % data) - self.impulseCallBack(data) + msg = BitStream() + msg.fromBytes(data) + self.decodeHeader(msg) + # khanat-opennel-code/code/ryzom/client/src/network_connection.cpp:bool CNetworkConnection::stateSynchronize() + message = msg.readUint8() + self.log.debug("_CurrentReceivedNumber:%d (mode:%s) %d [%d/%d/%d]" % (self._CurrentReceivedNumber, str(self._SystemMode), message, msg.sizeData(), msg.sizeRead(), msg.needRead())) + if message == 1: # SYSTEM_SYNC_CODE + self.log.debug("synchronize->synchronize") + self.receiveSystemSync(msg) + elif message == 3: # SYSTEM_PROBE_CODE + self.log.debug("synchronize->probe") + self._ConnectionState = TConnectionState.Probe + self.receiveSystemProbe(msg) + elif message == 6: # SYSTEM_STALLED_CODE + self.log.debug("received STALLED") + self._ConnectionState = TConnectionState.Stalled + self.receiveSystemStalled(msg) + elif message == 7: # SYSTEM_SERVER_DOWN_CODE + self.disconnect() + self.log.warning("BACK-END DOWN") + else: + self.log.warning("CNET: received system %d in state Synchronize" % message) + self.log.debug("_CurrentReceivedNumber:%d (mode:%s) %d [%d/%d/%d] '%s'" % (self._CurrentReceivedNumber, str(self._SystemMode), message, msg.sizeData(), msg.sizeRead(), msg.needRead(), msg.showLastData())) + + # self.impulseCallBack(data) self.log.info("Client Logout") self.sendSystemQuit() @@ -1443,7 +1621,9 @@ class ClientNetworkConnection: class ClientKhanat: def __init__(self, - khanaturl, + khanat_host, + khanat_port_login = 40916, + khanat_port_frontend = 47851, login="tester", password="tester", clientApp="Lirria", @@ -1451,7 +1631,8 @@ class ClientKhanat: url="/login/r2_login.php", suffix = None, download_patch = False, - show_patch_detail=False): + show_patch_detail=False, + size_buffer_file=1024): self.log = logging.getLogger('myLogger') if suffix is None: @@ -1460,7 +1641,9 @@ class ClientKhanat: self.download_patch = download_patch self.show_patch_detail = show_patch_detail - self.khanaturl = khanaturl + self.khanat_host = khanat_host + self.khanat_port_login = khanat_port_login + self.khanat_port_frontend = khanat_port_frontend self.login = login + suffix self.password = password self.clientApp = clientApp @@ -1471,25 +1654,20 @@ class ClientKhanat: self.log.debug("Temporary directory:%s" % self.tempdir) self.khanat_idx = CPersistentDataRecord(self.log) self.UserAddr, self.UserKey, self.UserId = None, None, None - self.clientNetworkConnection = ClientNetworkConnection(khanaturl) + self.clientNetworkConnection = ClientNetworkConnection(self.khanat_host, self.khanat_port_frontend) + self.size_buffer_file = size_buffer_file + self.cFileContainer = CFileContainer() def createAccount(self): - khanaturl = self.khanaturl.strip('"').strip("'") - try: - host, port = khanaturl.split(':') - except: - host = khanaturl - port = 40916 - - conn = http.client.HTTPConnection(host=host, port=port) + conn = http.client.HTTPConnection(host=self.khanat_host, port=self.khanat_port_login) cmd = "/ams/index.php?page=register" headers = {'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', 'Accept-Language' : 'en-US', 'Connection': 'keep-alive', 'DNT': '1', 'Cookie': 'PHPSESSID=lsoumn9f0ljgm3vo3hgjdead03', - 'Host': khanaturl+':'+ str(port), - 'Referer': 'http://' + khanaturl+':'+ str(port) + '/ams/index.php?page=register', + 'Host': self.khanat_host+':'+ str(self.khanat_port_login), + 'Referer': 'http://' + self.khanat_host+':'+ str(self.khanat_port_login) + '/ams/index.php?page=register', 'Upgrade-Insecure-Requests': '1', 'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; rv:6.0) Gecko/20100101 Firefox/6.0', 'Content-Type': 'application/x-www-form-urlencoded'} @@ -1533,14 +1711,7 @@ class ClientKhanat: self.log.info("Reuse account : %s" % self.login) def connectR2(self): - khanaturl = self.khanaturl.strip('"').strip("'") - try: - host, port = khanaturl.split(':') - except: - host = khanaturl - port = 40916 - - conn = http.client.HTTPConnection(host=host, port=port) + conn = http.client.HTTPConnection(host=self.khanat_host, port=self.khanat_port_login) cmd = self.url + "?cmd=ask&cp=2&login=" + self.login + "&lg=" + self.LanguageCode conn.request("GET", cmd) response = conn.getresponse() @@ -1563,7 +1734,7 @@ class ClientKhanat: cryptedPassword = crypt.crypt(self.password, salt) - conn = http.client.HTTPConnection(host=host, port=port) + conn = http.client.HTTPConnection(host=self.khanat_host, port=self.khanat_port_login) cmd = self.url + "?cmd=login&login=" + self.login + "&password=" + cryptedPassword + "&clientApplication=" + self.clientApp + "&cp=2" + "&lg=" + self.LanguageCode conn.request("GET", cmd) response = conn.getresponse() @@ -1614,7 +1785,7 @@ class ClientKhanat: break self.log.debug("size:%d", file_size) file_size_dl = 0 - block_size = 1024 # 8192 + block_size = self.size_buffer_file # 1024 with open(dest, 'wb') as fp: while True: @@ -1651,6 +1822,65 @@ class ClientKhanat: data = fin.read() fout.write(data) self.log.info("%s" % dstName) + os.remove(tmp) + # khanat-opennel-code/code/ryzom/client/src/login_patch.cpp # void CCheckThread::run () + FilesToPatch = [] + for file in self.khanat_idx.CBNPFile: + FilesToPatch.append(file) + # Here we got all the files to patch in FilesToPatch and all the versions that must be obtained Now we have to get the optional categories + OptionalCat = [] + for category in self.khanat_idx.Categories: + if category._IsOptional: + for file in category._Files: + bAdded = False + for file2 in FilesToPatch: + if file2 == file: + OptionalCat.append(category._Name) + bAdded = True + break + if bAdded: + break + # For all categories that required an optional category if the cat required is present the category that reference it must be present + for category in self.khanat_idx.Categories: + if category._IsOptional and not len(category._CatRequired) == 0: + bFound = False + for cat in OptionalCat: + if category._Name == cat: + bFound = True + break + if bFound: + for cat in OptionalCat: + if category._CatRequired == cat: + OptionalCat.append(category._Name) + break + # Delete categories optional cat that are hidden + for category in self.khanat_idx.Categories: + if category._IsOptional and category._Hidden: + for cat in OptionalCat: + if category._Name == cat: + OptionalCat.remove(category._Name) + break + # Get all extract to category and check files inside the bnp with real files + for category in self.khanat_idx.Categories: + if len(category._UnpackTo) != 0: + for file in category._Files: + # TODO + # readHeader() + pass + + def DownloadMinimum(self): + self.log.debug("-" * 80) + for file in self.khanat_idx.CBNPFile: + if file.FileName != "kh_server.bnp": + continue + tmp = self.getServerFile("%05d/%s.lzma" % (int(self.r2serverversion), file.FileName), False, "") + with lzma.open(tmp) as fin: + dstName = os.path.join(self.tempdir.name, file.FileName) + with open(dstName, "wb") as fout: + data = fin.read() + fout.write(data) + self.log.info("%s" % dstName) + os.remove(tmp) def Emulate(self): self.createAccount() @@ -1667,6 +1897,16 @@ class ClientKhanat: # Download all file in patch - login_patch.cpp:2578 # void CPatchThread::processFile (CPatchManager::SFileToPatch &rFTP) if self.download_patch: self.downloadAllPatch() + else: + self.DownloadMinimum() + self.cFileContainer = CFileContainer() + self.cFileContainer.addSearchPath(self.tempdir.name) + data = self.cFileContainer.getdata("msg.xml").decode() + msgXml = ET.fromstring(data) + ET.dump(msgXml) + data = self.cFileContainer.getdata("database.xml").decode() + databaseXml = ET.fromstring(data) + ET.dump(databaseXml) self.clientNetworkConnection.EmulateFirst() @@ -1676,11 +1916,15 @@ def main(): log = logging.getLogger('myLogger') parser = argparse.ArgumentParser() - parser.add_argument("--khanaturl", help="khanat URL to auhtenticate", default='localhost') + parser.add_argument("--khanat-host", help="khanat host to auhtenticate", default='localhost') parser.add_argument("--suffix", help="define suffix") parser.add_argument("-d", "--debug", help="show debug message", action='store_true') parser.add_argument("-p", "--download-patch", help="show debug message", action='store_true') parser.add_argument("-s", "--show-patch-detail", help="show debug message", action='store_true') + parser.add_argument("--size-buffer-file", help="size buffer to download file", type=int, default=1024) + parser.add_argument("--khanat-port-login", help="port http login", type=int, default=40916) + parser.add_argument("--khanat-port-frontend", help="port UDP frontend", type=int, default=47851) + args = parser.parse_args() if args.debug: @@ -1689,7 +1933,7 @@ def main(): level = logging.getLevelName('INFO') log.setLevel(level) - client = ClientKhanat(args.khanaturl, suffix=args.suffix, download_patch=args.download_patch, show_patch_detail=args.show_patch_detail) + client = ClientKhanat(args.khanat_host, khanat_port_login=args.khanat_port_login, khanat_port_frontend=args.khanat_port_frontend, suffix=args.suffix, download_patch=args.download_patch, show_patch_detail=args.show_patch_detail, size_buffer_file=args.size_buffer_file) client.Emulate() log.info("End")