ryzom_log_cleaner/main.py

#!/usr/bin/env python3

import tkinter as tk
from tkinter import ttk
from tkinter import filedialog
from tk_tooltip import CreateToolTip
import re
import os
import platform

channel_names = [
    "SAY",           # 0
    "SHOUT",         # 1
    "TEAM",          # 2
    "GUILD",         # 3
#    "CIVILIZATION", # Unused
#    "TERRITORY",    # Unused
    "UNIVERSE",      # 4
    "TELL",          # 5
#    "PLAYER",       # Unused
#    "ARROUND",      # Unused
    "REGION",        # 6
    "DYN0",          # 7
    "DYN1",          # 8
    "DYN2",          # 9
    "DYN3",          # 10
    "DYN4",          # 11
    "EMOTES",        # 12
    "SYSTEM",        # 13
]


system_info_categories = [
    ("SYS", "Default system messages"),
    ("BC", "Broadcast messages"),
    ("TAGBC", "Tagged Broadcast messages"),
    ("XP", "XP Gain"),
    ("SP", "SP Gain"),
    ("TTL", "Title"),
    ("TSK", "Task"),
    ("ZON", "Zone"),
    ("DG", "Damage to me"),
    ("DMG", "Damage to me"),
    ("DGP", "Damage to me from player"),
    ("DGM", "Damage from me"),
    ("MIS", "Opponent misses"),
    ("MISM", "I miss"),
    ("ITM", "Item"),
    ("ITMO", "Item other in group"),
    ("ITMF", "Item failed"),
    ("SPL", "Spell to me"),
    ("SPLM", "Spell from me"),
    ("EMT", "Emote"),
    ("MTD", "Message of the day"),
    ("FORLD", "Forage locate deposit"),
    ("CHK", "Failed check"),
    ("CHKCB", "Failed check in combat"),
    ("PVPTM", "PVP Timer"),
    ("THM", "Thema finished (encyclopedia)"),
    ("AMB", "Ambiance (Occupation)"),
    ("ISE", "Item special effect"),
    ("ISE2", "Item special effect centered text"),
    ("OSM", "Outpost state message"),
    ("AROUND", "Around channel system message"),
    ("R2_INVITE", "Ring invitation"),
]

say_1st_to_3rd_person = {
    "Vous": " dit ",
    "You": " says",
    "Du": " sagt",
}

class GUI:
    color_regex = re.compile('@\{[A-F0-9]{4}\}')

    def __init__(self):
        self.originalfolderlog = ''
        self.keep_color = False
        self.keep_channel = False
        self.keep_lang_flag = False
        self.keep_original_part = False
        self.keep_translated_part = False
        self.keep_timestamp = False
        self.replace_charname = False        
        
        self.window = tk.Tk()
        self.window.title("Ryzom Log Cleaner")

        self.window.columnconfigure(0, weight=1)
        #self.window.rowconfigure([0,1], minsize=50)

        self.ntb_file_selection = ttk.Notebook(self.window)

        ### Single file tab
        self.frm_single_file = tk.Frame(self.window)
        self.frm_single_file.columnconfigure(0, weight=1)
        btn_input = tk.Button(self.frm_single_file, text="Input file", command=self.get_input_filepath)
        self.ent_input = tk.Entry(self.frm_single_file, text="Input path")
    #    ent_input.insert(0, "~/log.txt")
        btn_output = tk.Button(self.frm_single_file, text="Output file", command=self.get_output_filepath)
        self.ent_output = tk.Entry(self.frm_single_file, text="Output path")
    #    ent_output.insert(0, "~/cleaned/")
        self.frm_charname = tk.Frame(self.frm_single_file)
        lbl_charname = tk.Label(self.frm_charname, text="Char name:")
        self.ent_charname = tk.Entry(self.frm_charname, text="Charname")
        self.ent_charname.insert(0, "Select input file to auto-fill")

        btn_input.grid(row=0, column=1, sticky='ew')
        self.ent_input.grid(row=0, column=0, sticky='ew')
        btn_output.grid(row=1, column=1, sticky='ew')
        self.ent_output.grid(row=1, column=0, sticky='ew')

        self.frm_charname.grid(row=2, column=0, columnspan=2, sticky='ew')
        self.frm_charname.columnconfigure(1, weight=1)

        lbl_charname.grid(row=0, column=0)
        self.ent_charname.grid(row=0, column=1, sticky='ew')

        
        ### Multi file tab
        self.frm_multi_file = tk.Frame(self.window)
        self.frm_multi_file.columnconfigure(0, weight=1)
        btn_input_multi = tk.Button(self.frm_multi_file, text="Input files", command=self.get_input_filepaths)
        self.ent_input_multi = tk.Entry(self.frm_multi_file, text="Input paths")
    #    ent_input.insert(0, "~/log.txt")
        btn_output_multi = tk.Button(self.frm_multi_file, text="Output directory", command=self.get_output_directory)
        self.ent_output_multi = tk.Entry(self.frm_multi_file, text="Output paths")
    #    ent_output.insert(0, "~/cleaned/")
        self.frm_charname_multi = tk.Frame(self.frm_multi_file)
        lbl_charname_multi = tk.Label(self.frm_charname_multi, text="Char name (detected from file name of each file if empty):")
        self.ent_charname_multi = tk.Entry(self.frm_charname_multi, text="Charname Multi")

        btn_input_multi.grid(row=0, column=1, sticky='ew')
        self.ent_input_multi.grid(row=0, column=0, sticky='ew')
        btn_output_multi.grid(row=1, column=1, sticky='ew')
        self.ent_output_multi.grid(row=1, column=0, sticky='ew')

        self.frm_charname_multi.grid(row=2, column=0, columnspan=2, sticky='ew')
        self.frm_charname_multi.columnconfigure(1, weight=1)

        lbl_charname_multi.grid(row=0, column=0)
        self.ent_charname_multi.grid(row=0, column=1, sticky='ew')

        
        ### Organise logs tab
        self.frm_organise_logs = tk.Frame(self.window)
        self.frm_organise_logs.columnconfigure(0, weight=1)
        btn_input_orgalogs = tk.Button(self.frm_organise_logs, text="Input directory", command=self.get_input_dir_orgalogs)
        self.ent_input_orgalogs = tk.Entry(self.frm_organise_logs, text="Input dir")
    #    ent_input.insert(0, "~/log.txt")
        btn_output_orgalogs = tk.Button(self.frm_organise_logs, text="(Empty) Output directory", command=self.get_output_dir_orgalogs)
        self.ent_output_orgalogs = tk.Entry(self.frm_organise_logs, text="Output dir")
    #    ent_output.insert(0, "~/cleaned/")
        self.frm_charname_orgalogs = tk.Frame(self.frm_organise_logs)
        lbl_charname_orgalogs = tk.Label(self.frm_charname_orgalogs, text="Char name (detected from file name of each file if empty):")
        self.ent_charname_orgalogs = tk.Entry(self.frm_charname_orgalogs, text="Charname Orgalogs")

        btn_input_orgalogs.grid(row=0, column=1, sticky='ew')
        self.ent_input_orgalogs.grid(row=0, column=0, sticky='ew')
        btn_output_orgalogs.grid(row=1, column=1, sticky='ew')
        self.ent_output_orgalogs.grid(row=1, column=0, sticky='ew')

        self.frm_charname_orgalogs.grid(row=2, column=0, columnspan=2, sticky='ew')
        self.frm_charname_orgalogs.columnconfigure(1, weight=1)

        lbl_charname_orgalogs.grid(row=0, column=0)
        self.ent_charname_orgalogs.grid(row=0, column=1, sticky='ew')

        self.btn_organise = tk.Button(self.frm_organise_logs, text="Organise!", command=self.organise_logs)
        self.btn_organise.grid(row=10, column=0, columnspan=2, sticky='ew')



        ### Setup tab notebook
        self.ntb_file_selection.add(self.frm_single_file, text="Single file input/output")
        self.ntb_file_selection.add(self.frm_multi_file, text="Multi file input/output")
        self.ntb_file_selection.add(self.frm_organise_logs, text="Organise logs")
        self.ntb_file_selection.grid(row=0, column=0, columnspan=2, sticky='ew', ipady='2.5')
        def on_tab_change(event):
            tab = event.widget.tab('current')['text']
            if tab == "Single file input/output":
                self.btn_process.config(command=self.process_file)
                self.frm_process_settings.grid(in_=self.frm_single_file)
            elif tab == "Multi file input/output":
                self.btn_process.config(command=self.process_files)
                self.frm_process_settings.grid(in_=self.frm_multi_file)
        self.ntb_file_selection.bind('<<NotebookTabChanged>>', on_tab_change)


        self.frm_process_settings = tk.Frame(self.window)
        self.frm_process_settings.grid(row=5, column=0, columnspan=2, sticky='ew', in_=self.frm_single_file)
        self.frm_process_settings.columnconfigure([0,1], weight=1)


        sep_general_settings = ttk.Separator(self.frm_process_settings, orient='horizontal')
        sep_general_settings.grid(row=5, column=0, columnspan=2, sticky='ew', pady='5')

        ### General settings
        self.frm_general_settings = tk.Frame(self.frm_process_settings)
        self.frm_general_settings.grid(row=6, column=0, columnspan=2, sticky='ew', pady='5')
        self.frm_general_settings.columnconfigure([0,1,2], weight=1)
        
        self.btn_keep_color = self.create_toggle_button(self.frm_general_settings, "Keep color", self.toggle_setting("keep_color"))
        self.btn_keep_channel = self.create_toggle_button(self.frm_general_settings, "Keep channel name", self.toggle_setting("keep_channel"))
        self.btn_keep_lang_flag = self.create_toggle_button(self.frm_general_settings, "Keep language flag", self.toggle_setting("keep_lang_flag"))
        self.btn_keep_original_part = self.create_toggle_button(self.frm_general_settings, "Keep original text", self.toggle_setting("keep_original_part"))
        self.btn_keep_translated_part = self.create_toggle_button(self.frm_general_settings, "Keep translated text", self.toggle_setting("keep_translated_part"))
        self.btn_keep_timestamp = self.create_toggle_button(self.frm_general_settings, "Keep timestamp", self.toggle_setting("keep_timestamp"))
        self.btn_replace_charname = self.create_toggle_button(self.frm_general_settings, "Replace charname", self.toggle_setting("replace_charname"))

        self.btn_keep_color.grid(row=0, column=0, sticky='ew')
        self.btn_keep_channel.grid(row=0, column=1, sticky='ew')
        self.btn_keep_lang_flag.grid(row=0, column=2, sticky='ew')
        self.btn_keep_original_part.grid(row=1, column=0, sticky='ew')
        self.btn_keep_translated_part.grid(row=1, column=1, sticky='ew')
        self.btn_keep_timestamp.grid(row=1, column=2, sticky='ew')
        self.btn_replace_charname.grid(row=2, column=0, sticky='ew')
        
        sep_channels = ttk.Separator(self.frm_process_settings, orient='horizontal')
        sep_channels.grid(row=10, column=0, columnspan=2, sticky='ew')

        lbl_channel_select = tk.Label(self.frm_process_settings, text="Select channels to keep:")
        lbl_channel_select.grid(row=11, column=0, sticky='w')

        self.btn_chan_toggle = []

        self.frm_btn_channel = tk.Frame(self.frm_process_settings)
        self.frm_btn_channel.grid(row=12, column=0, columnspan=2, sticky='ew')
        self.frm_btn_channel.columnconfigure([0,1,2,3], weight=1)
        self.frm_btn_channel.rowconfigure([0,1,2], weight=1)
        for i,name in enumerate(channel_names):
            self.btn_chan_toggle.append(self.create_toggle_button(self.frm_btn_channel, name, self.toggle_channel(i)))
            self.btn_chan_toggle[i].grid(row=int(i/4), column=i%4, sticky='ew')

        self.btn_sys_toggle = []

        self.frm_btn_sys = tk.Frame(self.frm_process_settings, bg="#808080", borderwidth=5)
        self.frm_btn_sys.grid(row=13, column=0, columnspan=2, sticky='e')
        self.frm_btn_sys.columnconfigure([0,1,2,3,4,5,6], weight=1)
        self.frm_btn_sys.rowconfigure([0,1,2,3,4], weight=1)
        for i,(name,tooltip) in enumerate(system_info_categories):
            self.btn_sys_toggle.append(self.create_toggle_button(self.frm_btn_sys, name, self.toggle_system(i)))
            self.btn_sys_toggle[i].grid(row=int(i/7), column=i%7, sticky='ew')
            ttp_sys = CreateToolTip(self.btn_sys_toggle[i], tooltip)

        self.btn_process = tk.Button(self.frm_process_settings, text="Process!", command=self.process_file)
        self.btn_process.grid(row=20, column=0, columnspan=2, sticky='ew')

        # Default selection
        self.chan_toggle = [False] * len(channel_names)
        self.sys_toggle = [False] * len(system_info_categories)
        self.toggle_channel(0)()
        self.toggle_channel(1)()
        self.toggle_channel(12)()
        self.toggle_channel(13)()
        self.toggle_system(7)()
        self.toggle_setting("keep_translated_part")()

    def create_toggle_button(self, parent, text, command):
        btn = None
        if platform.system() == "Darwin":
            btn = tk.Label(parent, text=text, bg="#ffcccb", relief='raised')
            btn.bind("<Button-1>",lambda e:command())
            btn.grid(padx=1, pady=1, ipady=3)
        else:
            btn = tk.Button(parent, text=text, command=command, bg="#ffcccb")
        return btn

        
    def toggle_setting(self, setting):
        def tgl_pref():
            if getattr(self, setting):
                getattr(self, f"btn_{setting}").config(relief="raised", bg="#ffcccb")
                setattr(self, setting, False)
            else:
                getattr(self, f"btn_{setting}").config(relief="sunken", bg="#99e599")
                setattr(self, setting, True)
        return tgl_pref

    def get_input_filepath(self):
        filepath = filedialog.askopenfilename(
            filetypes=[("Text Files", "*.txt"), ("All Files", "*.*")],
            initialdir=os.path.dirname(self.ent_input.get())
        )
        if not filepath:
            return
        self.ent_input.delete(0, tk.END)
        self.ent_input.insert(0, filepath)
        filename = os.path.basename(filepath)
        name_start = filename.find("log_")
        name_end = filename.find("_", name_start+4)
        if name_end == -1:
            name_end = filename.find(".", name_start+4)
        if name_start != -1:
            self.ent_charname.delete(0, tk.END)
            self.ent_charname.insert(0, os.path.basename(filepath)[name_start+4:name_end])

    def get_output_filepath(self):
        filepath = filedialog.asksaveasfilename(
            filetypes=[("Text Files", "*.txt"), ("All Files", "*.*")],
            initialdir=os.path.dirname(self.ent_output.get())
        )
        if not filepath:
            return
        self.ent_output.delete(0, tk.END)
        self.ent_output.insert(0, filepath)

    def get_input_filepaths(self):
        filepaths = filedialog.askopenfilenames(
            filetypes=[("Text Files", "*.txt"), ("All Files", "*.*")],
            initialdir=os.path.dirname(self.ent_input_multi.get().split(';')[0])
        )
        if not filepaths:
            return
        self.ent_input_multi.delete(0, tk.END)
        self.ent_input_multi.insert(0, ';'.join(filepaths))

    def get_output_directory(self):
        directory = filedialog.askdirectory(
            initialdir=self.ent_output_multi.get()
        )
        if not directory:
            return
        self.ent_output_multi.delete(0, tk.END)
        self.ent_output_multi.insert(0, directory)

    def get_input_dir_orgalogs(self):
        directory = filedialog.askdirectory(
            initialdir=self.ent_input_orgalogs.get()
        )
        if not directory:
            return
        self.ent_input_orgalogs.delete(0, tk.END)
        self.ent_input_orgalogs.insert(0, directory)

    def get_output_dir_orgalogs(self):
        directory = filedialog.askdirectory(
            initialdir=self.ent_output_orgalogs.get()
        )
        if not directory:
            return
        self.ent_output_orgalogs.delete(0, tk.END)
        self.ent_output_orgalogs.insert(0, directory)

        
    def toggle_channel(self, index):
        def tgl_chan():
            if self.chan_toggle[index]:
                self.btn_chan_toggle[index].config(relief="raised", bg="#ffcccb")
                self.chan_toggle[index] = False
                if channel_names[index] == "SYSTEM":
                    self.frm_btn_sys.grid_remove()
            else:
                self.btn_chan_toggle[index].config(relief="sunken", bg="#99e599")
                self.chan_toggle[index] = True
                if channel_names[index] == "SYSTEM":
                    self.frm_btn_sys.grid()
        return tgl_chan

    def toggle_system(self,index):
        def tgl_sys():
            if self.sys_toggle[index]:
                self.btn_sys_toggle[index].config(relief="raised", bg="#ffcccb")
                self.sys_toggle[index] = False
            else:
                self.btn_sys_toggle[index].config(relief="sunken", bg="#99e599")
                self.sys_toggle[index] = True
        return tgl_sys

    def process_file(self):
        if not self.check_path_exists("ent_input"):
            return
        chan_pattern = '|'.join(['\(' + name + '\)' for i,name in enumerate(channel_names[:-2]) if self.chan_toggle[i]])
        if self.chan_toggle[-2]:
            chan_pattern += '|\(SAY/EMT\)'
        if self.chan_toggle[-1]:
            if self.sys_toggle[0]:
                chan_pattern += '|\(SYSTEM\)'
            for i,(name,tooltip) in enumerate(system_info_categories[1:]):
                if self.sys_toggle[i+1]:
                    chan_pattern += '|\(SYSTEM/' + name + '\)'
        chan_regex = re.compile(chan_pattern)
        with open(self.ent_input.get(), 'r', errors='surrogateescape') as in_file, open(self.ent_output.get(), 'w', errors='surrogateescape') as out_file:
            orig_lines = 0
            filtered_lines = 0
            self.btn_process["text"] = "Started Processing..."
            for line in in_file:
                orig_lines += 1
                if chan_regex.search(line) == None:
                    continue
                if not self.keep_color:
                    line = self.color_regex.sub('',line)
                if not self.keep_channel:
                    line = line[:20] + line[line.find(') * ')+1:]
                if not self.keep_timestamp:
                    line = line[20:]
                original_start = line.find('{:')
                if original_start != -1 and not self.keep_lang_flag:
                    line = line[:original_start+1] + line[original_start+5:]
                original_end = line.find('}@{')
                if original_end != -1 and not self.keep_translated_part:
                    line = line[:original_end+4] + '\n'
                if original_end != -1 and not self.keep_original_part:
                    original_text_start = original_start + (5 if self.keep_lang_flag else 1)
                    line = line[:original_text_start] + line[original_end:]
                if original_end != -1:
                    original_end = line.find('}@{')
                    line = line[:original_start] + line[original_start+1:original_end] + line[original_end+4:]
                    
                if self.replace_charname:
                    char_name_start = line.find(' * ') + 3 if not self.keep_color or line.find('}') == -1 else line.find('}') + 1
                    if line[char_name_start] == '[' and line[char_name_start+2] == ']':
                        char_name_start += 3
                    char_name_end = line.find(':', char_name_start)
                    char_talks_end = char_name_end
                    char_name_end -= len(line[:char_name_end].rstrip().split(' ')[-1]) + 1 + (len(line[:char_name_end])-len(line[:char_name_end].rstrip()))
                    char_name = line[char_name_start:char_name_end]
                    
                    if char_name in say_1st_to_3rd_person:
                        line = line[:char_name_start] + ' '.join([s.capitalize() for s in self.ent_charname.get().split(' ')]) + say_1st_to_3rd_person[char_name] + line[char_talks_end:]

                line = line[:line.find(' * ')] + line[line.find(' * ')+3:]
    #            channel_name = line[21:line.find(')')]
                out_file.write(line.lstrip())
                filtered_lines += 1
            self.btn_process["text"]="Processing done! (" + str(filtered_lines) + " lines kept out of " + str(orig_lines) + " original lines)"

    def process_files(self):        
        input_filepaths = self.ent_input_multi.get().split(';')
        for filepath in input_filepaths:
            filename = os.path.basename(filepath)
            self.ent_input.delete(0, tk.END)
            self.ent_input.insert(0, filepath)
            self.ent_output.delete(0, tk.END)
            self.ent_output.insert(0, os.path.join(self.ent_output_multi.get(), filename))
            if len(self.ent_charname_multi.get()) == 0:
                name_start = filename.find("log_")
                name_end = filename.find("_", name_start+4)
                if name_end == -1:
                    name_end = filename.find(".", name_start+4)
                    if name_start != -1:
                        self.ent_charname.delete(0, tk.END)
                        self.ent_charname.insert(0, os.path.basename(filepath)[name_start+4:name_end])
            else:
                self.ent_charname.delete(0, tk.END)
                self.ent_charname.insert(0, self.ent_charname_multi.get())
            
            self.process_file()

        
    def organise_logs(self):
        if not self.check_path_exists("ent_output_orgalogs") or not self.check_path_exists("ent_input_orgalogs"):
            return
        self.my_pathes = {}
        for dirpath, _, filenames in os.walk(self.ent_input_orgalogs.get()):
            for f in filenames:
                with open(os.path.join(dirpath, f), 'r', errors='surrogateescape') as in_f:
                    last_year = -1
                    last_month = -1
                    last_day = -1
                    out_f = None
                    if len(self.ent_charname_orgalogs.get()) == 0:
                        name_start = f.find("log_")
                        name_end = f.find("_", name_start+4)
                        if name_end == -1:
                            name_end = f.find(".", name_start+4)
                            if name_start != -1:
                                self.ent_charname_orgalogs.delete(0, tk.END)
                                self.ent_charname_orgalogs.insert(0, f[name_start+4:name_end])
                    charname = self.ent_charname_orgalogs.get().lower()
                    manual_check_p = self.make_path_safe(os.path.join(self.ent_output_orgalogs.get(), f"log_{charname}_manual_check.log"))
                    manual_check_f = open(manual_check_p, 'a', errors='surrogateescape')
                    for line in in_f:
                        year_end = line.find('/')
                        unclassified = False
                        year = month = day = 0
                        if year_end == -1 or year_end+3 >= len(line) or line[year_end+3] != '/':
                            if last_year != -1:
                                year, month, day = last_year, last_month, last_day
                            else:
                                unclassified = True
                        else:
                            try:
                                year = int(line[:year_end])
                                month = int(line[year_end+1:year_end+3])
                                day = int(line[year_end+4:year_end+6])
                            except ValueError:
                                unclassified = True
                        if not unclassified and (year != last_year or month != last_month or day != last_day):
                            if out_f != None:
                                out_f.close()
                            out_p = self.make_path_safe(os.path.join(self.ent_output_orgalogs.get(), charname, f"{year:04d}", f"{month:02d}", f"log_{charname}_{year:04d}_{month:02d}_{day:02d}.log"))
                            out_f = open(out_p, 'a', errors='surrogateescape')
                        if unclassified:
                            manual_check_f.write(line)
                        else:
                            out_f.write(line)
                            last_year = year
                            last_month = month
                            last_day = day
                    if out_f != None:
                        out_f.close()
                    manual_check_f.close()
                    
    def make_path_safe(self, path):
        if path in self.my_pathes:
            return self.my_pathes[path]
        (dirpath, filename) = os.path.split(path)
        (basefile, ext) = os.path.splitext(filename)
        if ext == '':
            os.makedirs(path, exist_ok=True)
            self.my_pathes[path] = path
            return path
        else:
            os.makedirs(dirpath, exist_ok=True)
            counter = 1
            new_path = path
            while os.path.exists(new_path):
                new_path = os.path.join(dirpath, f"{basefile}_{counter}{ext}")
                counter += 1
            self.my_pathes[path] = new_path
            return new_path
                
                    
    def check_path_exists(self, attribute):
        if not hasattr(self, attribute) or getattr(self, attribute) == None or not os.path.exists(getattr(self, attribute).get()):
            tk.messagebox.showerror(title="Error in files selection", message=f"Error with input/output files. Make sur the input/output files or directories are set and exist")
            return False
        return True

        

if __name__ == '__main__':
    gui = GUI()
    
    gui.window.mainloop()