################################################################################
#
#  Copyright 2014-2016 Eric Lacombe <eric.lacombe@security-labs.org>
#
################################################################################
#
#  This file is part of fuddly.
#
#  fuddly is free software: you can redistribute it and/or modify
#  it under the terms of the GNU General Public License as published by
#  the Free Software Foundation, either version 3 of the License, or
#  (at your option) any later version.
#
#  fuddly is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#  GNU General Public License for more details.
#
#  You should have received a copy of the GNU General Public License
#  along with fuddly. If not, see <http://www.gnu.org/licenses/>
#
################################################################################

import os
import time
import traceback
import sys
import datetime
import threading
import itertools
import logging

from typing import List, Tuple

from fuddly.framework.scenario import Scenario
from fuddly.libs.external_modules import *
from fuddly.libs.utils import get_caller_object
from fuddly.framework.data import Data, DataAttr
from fuddly.framework.global_resources import *
from fuddly.framework.database import Database
from fuddly.framework.knowledge.feedback_collector import FeedbackSource
from fuddly.libs.utils import ExternalDisplay, Accumulator
from fuddly.framework import global_resources as gr

class Logger(object):
    """
    The Logger is used for keeping the history of the communication
    with the Target. The methods are used by the framework, but can
    also be leveraged by a Director.
    """

    fmkDB = None

    FLUSH_API = 1
    WRITE_API = 2
    PRETTY_PRINT_API = 3
    PRINT_CONSOLE_API = 4

    STATUS_API = 10
    MARKUP_API = 11
    ANYFIFO_API = 12
    BASIC_OUTPUT_API = 13

    def __init__(
        self,
        name=None,
        prefix="",
        record_data=False,
        explicit_data_recording=False,
        export_raw_data=True,
        term_display_limit=800,
        enable_term_display=True,
        enable_file_logging=False,
        highlight_marked_nodes=False,
    ):
        """
        Args:
          name (str): Name to be used in the log filenames. If not specified, the name of the project
            in which the logger is embedded will be used.
          record_data (bool): If True, each emitted data will be stored in a specific
            file within `exported_data/`.
          explicit_data_recording (bool): Used for logging outcomes further to a Director instruction. If True,
            the director would have to state explicitly if it wants the just emitted data to be recorded.
            Such notification is possible when the framework call its method
            :meth:`framework.director_helpers.Director.do_after_all()`, where the Director can take its decision
            after the observation of the target feedback and/or probes outputs.
          export_raw_data (bool): If True, will log the data as it is, without trying to interpret it
            as human readable text.
          term_display_limit (int): maximum amount of characters to display on the terminal at once.
            If this threshold is overrun, the message to print on the console will be truncated.
          enable_term_display (bool): If True, information will be displayed on the terminal
          prefix (str): prefix to use for printing on the console.
          enable_file_logging (bool): If True, file logging will be enabled.
          highlight_marked_nodes (bool): If True, alteration performed by compatible operators
            will be highlighted. Only possible if `export_raw_data` is False, as this option forces
            data interpretation.
        """

        self.name = name
        self.p = prefix
        self.current_project = None
        self.__record_data = record_data
        self.__explicit_data_recording = explicit_data_recording
        self._term_display_limit = term_display_limit

        now = datetime.datetime.now()
        self.__prev_export_date = now.strftime("%Y%m%d_%H%M%S")
        self.__export_cpt = 0
        self.export_raw_data = export_raw_data

        self._enable_file_logging = enable_file_logging
        self._fd = None

        if export_raw_data and highlight_marked_nodes:
            raise ValueError(
                "When @highlight_marked_nodes is True, @export_raw_data should be False"
            )

        self._hl_marked_nodes_user_provided_value = highlight_marked_nodes
        self._hl_marked_nodes = highlight_marked_nodes

        self._tg_fbk = []
        self._tg_fbk_lck = threading.Lock()

        self.display_on_term = enable_term_display

        self._log_handler_thread = None
        self._log_handler_stop_event = threading.Event()

        self._thread_initialized = threading.Event()
        self._log_entry_submitted_cond = threading.Condition()
        self._log_entry_list = []
        # self._log_displayed = threading.Event()
        self._sync_lock = threading.Lock()

        self._ext_disp = ExternalDisplay()

        self.style2bbcode = {
            None: '',
            FontStyle.BOLD: 'bold',
        }

        self._post_processed_info = None

        def init_logfn(
                x,
                nl_before=True,
                nl_after=False,
                rgb=None,
                style=None,
                verbose=False,
                do_record=True,
                basic_output = False,
                all_output= False
        ):
            if not self.display_on_term:
                return

            no_format_mode = False
            if issubclass(x.__class__, Data):
                data = self._handle_binary_content(
                    x.to_bytes(), raw=self.export_raw_data
                )
                colored_data = x.to_formatted_str() if self._hl_marked_nodes else data
                rgb = None
                style = None
                no_format_mode = self._hl_marked_nodes
            elif isinstance(x, str):
                colored_data = data = x
            else:
                colored_data = data = self._handle_binary_content(
                    x, raw=self.export_raw_data
                )
            self.print_console(
                colored_data,
                nl_before=nl_before,
                nl_after=nl_after,
                rgb=rgb,
                style=style,
                no_format_mode=no_format_mode,
                basic_output=basic_output,
                all_output=all_output
            )
            if verbose and issubclass(x.__class__, Data):
                self.pretty_print_data(x)

            return data

        self.log_fn = init_logfn

    def switch_log_format(self):
        self.export_raw_data = not self.export_raw_data
        if not self.export_raw_data:
            self._hl_marked_nodes = self._hl_marked_nodes_user_provided_value
        else:
            self._hl_marked_nodes = False

    def set_log_format(self, raw: bool):
        self.export_raw_data = raw
        if not self.export_raw_data:
            self._hl_marked_nodes = self._hl_marked_nodes_user_provided_value
        else:
            self._hl_marked_nodes = False

    def flush(self):
        with self._sync_lock:
            with self._log_entry_submitted_cond:
                self._log_entry_list.append((Logger.FLUSH_API, None))
                self._log_entry_submitted_cond.notify()

    def write(self, data: str):
        with self._sync_lock:
            with self._log_entry_submitted_cond:
                self._log_entry_list.append((Logger.WRITE_API, data))
                self._log_entry_submitted_cond.notify()

    def pretty_print_data(self, data: Data, fd=None, raw_limit: int = None, verbose=Verbose.Heavy, debug=False):
        with self._sync_lock:
            with self._log_entry_submitted_cond:
                self._log_entry_list.append(
                    (Logger.PRETTY_PRINT_API, (data, fd, raw_limit, verbose, debug))
                )
                self._log_entry_submitted_cond.notify()

    def print_status(self, status_msg: str):
        with self._sync_lock:
            with self._log_entry_submitted_cond:
                self._log_entry_list.append(
                    (Logger.STATUS_API, status_msg)
                )
                self._log_entry_submitted_cond.notify()

    def print_markup(self, markup_msg: str):
        with self._sync_lock:
            with self._log_entry_submitted_cond:
                self._log_entry_list.append(
                    (Logger.MARKUP_API, markup_msg)
                )
                self._log_entry_submitted_cond.notify()

    def print_main(self, msg: str):
        with self._sync_lock:
            with self._log_entry_submitted_cond:
                self._log_entry_list.append(
                    (Logger.BASIC_OUTPUT_API, msg)
                )
                self._log_entry_submitted_cond.notify()

    def print_on(self, fifo, data: str, newline=True):
        with self._sync_lock:
            with self._log_entry_submitted_cond:
                self._log_entry_list.append(
                    (Logger.ANYFIFO_API, (fifo, data, newline))
                )
                self._log_entry_submitted_cond.notify()

    def set_external_display(self, disp):
        self._ext_disp = disp

    def __str__(self):
        return "Logger"

    def _handle_binary_content(self, content, raw=False):
        content = gr.unconvert_from_internal_repr(content)
        content = content if not raw else "{!a}".format(content)

        return content

    def start(self, tui=False):
        self.__idx = 0
        self.__tmp = False
        self._tui = tui

        self.reset_current_state()
        self._current_sent_date = None
        self._last_data_IDs = {}  # per target_ref
        self._returned_last_data_ID = None
        self._returned_last_data_attrs = None
        self.last_data_recordable = None

        self._post_processed_info = None

        with self._tg_fbk_lck:
            self._tg_fbk = []

        if self.name is None:
            raise ValueError(
                "Logger's name shall be provided (either by the framework or at init)"
            )

        elif self._enable_file_logging:
            self.now = datetime.datetime.now()
            self.now = self.now.strftime("%Y_%m_%d_%H%M%S")

            log_file = os.path.join(logs_folder, self.now + "_" + self.name + "_log")
            self._fd = open(log_file, "w")

            def intern_func(
                    x,
                    nl_before=True,
                    nl_after=False,
                    rgb=None,
                    style=None,
                    verbose=False,
                    do_record=True,
                    basic_output=False,
                    all_output=False,
            ):
                no_format_mode = False
                if issubclass(x.__class__, Data):
                    data = self._handle_binary_content(
                        x.to_bytes(), raw=self.export_raw_data
                    )
                    colored_data = (
                        x.to_formatted_str() if self._hl_marked_nodes else data
                    )
                    rgb = None
                    style = None
                    no_format_mode = self._hl_marked_nodes
                elif isinstance(x, str):
                    colored_data = data = x
                else:
                    colored_data = data = self._handle_binary_content(
                        x, raw=self.export_raw_data
                    )
                self.print_console(
                    colored_data,
                    nl_before=nl_before,
                    nl_after=nl_after,
                    rgb=rgb,
                    style=style,
                    no_format_mode=no_format_mode,
                    basic_output=basic_output,
                    all_output=all_output
                )
                if not do_record:
                    return data
                try:
                    self._fd.write(data + "\n")
                    if verbose and issubclass(x.__class__, Data):
                        self.pretty_print_data(x, fd=self._fd)
                    self._fd.flush()
                except ValueError:
                    self.print_console(
                        "\n*** ERROR: The log file has been closed."
                        " (Maybe because the Logger has been stopped and has not been restarted yet.)",
                        rgb=Color.ERROR,
                        basic_output=True
                    )

                return data

            self.log_fn = intern_func

        else:
            # No file logging
            pass

        if self._log_handler_thread is not None:
            return

        self._log_handler_thread = threading.Thread(
            None, self._log_handler, "log_handler"
        )
        self._log_handler_thread.start()
        while not self._thread_initialized.is_set():
            self._thread_initialized.wait(0.1)

        # if self._tui:
        #     bbcode = Color.to_bbcode(Color.COMPONENT_START)
        #     self._ext_disp.disp.set_basic_output_mode(markup=True)
        #     time.sleep(0.1)
        #     self.print_main(f"[{bbcode}]*** Logger is started ***[/]\n")
        #     self._ext_disp.disp.set_basic_output_mode(markup=False)
        # else:
        self.print_console(
            "*** Logger is started ***\n", nl_before=False, rgb=Color.COMPONENT_START,
            basic_output=True
        )

    def _stop_log_handler(self):
        with self._sync_lock:
            self._log_handler_stop_event.set()
            self._log_handler_thread.join()
            self._log_handler_stop_event.clear()

    def _log_handler(self):

        sys.stdout.write(colorize('\n*** The display server thread has started ***\n', rgb=Color.FMKINFO))

        try:
            self._thread_initialized.set()
            accu = Accumulator()

            while True:
                with self._log_entry_submitted_cond:
                    if self._log_handler_stop_event.is_set() and not self._log_entry_list:
                        break
                    self._log_entry_submitted_cond.wait(0.001)

                    if self._log_entry_list:
                        log_entries = self._log_entry_list
                        self._log_entry_list = []
                    else:
                        continue

                for log_e in log_entries:
                    api, params = log_e

                    if api == Logger.FLUSH_API:
                        if not self._ext_disp.is_enabled:
                            sys.stdout.flush()
                    elif api == Logger.WRITE_API:
                        if self._ext_disp.is_enabled:
                            self._ext_disp.disp.print(params)
                        else:
                            sys.stdout.write(params)
                    elif api == Logger.STATUS_API:
                        if self._ext_disp.is_enabled:
                            self._ext_disp.disp.print_status(params)
                        else:
                            pass
                    elif api == Logger.MARKUP_API:
                        if self._ext_disp.is_enabled:
                            self._ext_disp.disp.print_markup(params)
                        else:
                            sys.stdout.write(params)
                    elif api == Logger.BASIC_OUTPUT_API:
                        if self._ext_disp.is_enabled:
                            self._ext_disp.disp.print_main(params)
                        else:
                            sys.stdout.write(params + '\n')
                    elif api == Logger.ANYFIFO_API:
                        if self._ext_disp.is_enabled:
                            self._ext_disp.disp.print_on(*params)
                        else:
                            pass
                    elif api == Logger.PRETTY_PRINT_API:
                        data, fd, raw_limit, verbose, debug = params
                        if fd is None:
                            try:
                                data.show(log_func=accu.accumulate, raw_limit=raw_limit, verbose=verbose,
                                          debug=debug)
                            except Exception as e:
                                err_msg = colorize(f"\n*** ERROR: data.show() raise the exception [{e}]\n",
                                                   rgb=Color.ERROR)
                                if self._ext_disp.is_enabled:
                                    self._ext_disp.disp.print(err_msg)
                                else:
                                    sys.stderr.write(err_msg)
                                traceback.print_exc()

                            if self._ext_disp.is_enabled:
                                self._ext_disp.disp.print_main('\n' + accu.content)
                            else:
                                sys.stdout.write('\n' + accu.content)
                            accu.clear()
                        else:
                            try:
                                data.show(log_func=fd.write, raw_limit=raw_limit, verbose=verbose,
                                          debug=debug)
                            except Exception as e:
                                err_msg = colorize(f"\n*** ERROR: data.show() raise the exception [{e}]\n",
                                                   rgb=Color.ERROR)
                                fd.write(err_msg)
                                traceback.print_exc()

                            fd.flush()

                    elif api == Logger.PRINT_CONSOLE_API:
                        self._print_console(*params)
                    else:
                        self._print_console(
                            "*** ERROR[Logger]: Unknown API ***", rgb=Color.ERROR, basic_output=True
                        )
        except Exception as e:
            err_msg = colorize(f'\n*** ERROR: The display server thread got an exeception [{e}]\n',
                               rgb=Color.ERROR)
            sys.stderr.write(err_msg)
            traceback.print_exc()
        else:
            self._log_handler_stop_event.wait(0.001)
        finally:
            sys.stdout.write(colorize('\n*** The display server thread has stopped ***\n', rgb=Color.FMKINFO))

    def wait_for_sync(self):
        # while not self._log_displayed.is_set():
        #     self._log_displayed.wait(0.01)
        # self._log_displayed.clear()
        time.sleep(0.1)

    def stop(self):
        if self._fd:
            self._fd.close()

        self.current_project = None
        self.reset_current_state()
        self._current_sent_date = None
        self._last_data_IDs = {}
        self._returned_last_data_ID = None
        self._returned_last_data_attrs = None
        self.last_data_recordable = None
        self._stop_log_handler()
        self._log_handler_thread = None

        self.print_console(
            "*** Logger is stopped ***\n", nl_before=False, rgb=Color.COMPONENT_STOP
        )

    def reset_current_state(self):
        self._current_data = None
        self._current_group_id = None
        self._current_size = None
        self._current_ack_dates = None
        self._current_dmaker_list = []
        self._current_dmaker_info = {}
        self._current_src_data_id = None
        self._current_fmk_info = []

    def commit_data_table_entry(self, group_id, prj_name):
        if self._current_data is not None:  # that means data will be recorded
            init_dmaker = self._current_data.get_initial_dmaker()
            init_dmaker = (
                Database.DEFAULT_GTYPE_NAME if init_dmaker is None else init_dmaker[0]
            )
            dm = self._current_data.get_data_model()
            dm_name = Database.DEFAULT_DM_NAME if dm is None else dm.name
            self._current_group_id = group_id

            last_data_id = None
            d_orig = self._current_data.origin
            orig_ref = d_orig.reference if isinstance(d_orig, Scenario) else str(d_orig)
            for tg_ref, ack_date in self._current_ack_dates.items():
                last_data_id = self.fmkDB.insert_data(
                    init_dmaker,
                    dm_name,
                    self._current_data.to_bytes(),
                    self._current_size,
                    self._current_sent_date,
                    ack_date,
                    tg_ref,
                    prj_name,
                    group_id=group_id,
                    origin=orig_ref,
                    attrs=str(self._current_data.attrs)
                )
                # assert isinstance(tg_ref, FeedbackSource)
                self._last_data_IDs[tg_ref.obj] = last_data_id

                if last_data_id is None:
                    print("\n*** ERROR: Cannot insert the data record in FMKDB!")
                    self.last_data_recordable = None
                    return last_data_id

                self._current_data.set_data_id(last_data_id)

                step_id_start = 1

                for step_id, dmaker in enumerate(
                    self._current_dmaker_list, start=step_id_start
                ):
                    dmaker_type, dmaker_name, user_input = dmaker
                    info = self._current_dmaker_info.get(
                        (dmaker_type, dmaker_name), None
                    )
                    if info is not None:
                        info = "\n".join(info)
                        info = convert_to_internal_repr(info)
                    ui = str(user_input) if bool(user_input) else None
                    self.fmkDB.insert_steps(
                        last_data_id,
                        step_id,
                        dmaker_type,
                        dmaker_name,
                        self._current_src_data_id,
                        ui,
                        info,
                    )

                for msg, now in self._current_fmk_info:
                    self.fmkDB.insert_fmk_info(last_data_id, msg, now)

            self._returned_last_data_ID = last_data_id
            self._returned_last_data_attrs = self._current_data.attrs
            return last_data_id

        else:
            return None

    def mark_last_data_final(self):
        if self._returned_last_data_attrs is not None:
            self._returned_last_data_attrs.set(DataAttr.SC_FinalData)
            self.fmkDB.update_data(self._returned_last_data_ID,
                                   new_attrs=str(self._returned_last_data_attrs))


    def log_async_data(
        self,
        data_list: Data | List[Data] | Tuple[Data],
        sent_date,
        target_ref,
        prj_name,
        current_data_id,
    ):
        if isinstance(data_list, Data):
            data_list = (data_list,)

        for data in data_list:
            raw_data = data.to_bytes()
            data_sz = len(raw_data)
            init_dmaker = data.get_initial_dmaker()
            dtype = (
                Database.DEFAULT_GTYPE_NAME if init_dmaker is None else init_dmaker[0]
            )
            dm = data.get_data_model()
            dm_name = Database.DEFAULT_DM_NAME if dm is None else dm.name
            self.fmkDB.insert_async_data(
                dtype=dtype,
                dm_name=dm_name,
                raw_data=raw_data,
                sz=data_sz,
                sent_date=sent_date,
                target_ref=target_ref,
                prj_name=prj_name,
                current_data_id=current_data_id,
            )

    def log_fmk_info(
            self,
            info,
            nl_before=False,
            nl_after=False,
            rgb=Color.FMKINFO,
            data_id=None,
            do_show=True,
            do_record=True,
            delay_recording=False,
            deco = '***',
            basic_output = False,
            all_output = False,
    ):
        now = datetime.datetime.now()

        p = "\n" if nl_before else ""
        s = "\n" if nl_after else ""

        msg = f"{p:s}{deco} [ {info} ] {deco}{s:s}"
        if do_show:
            self.log_fn(msg, rgb=rgb, basic_output=basic_output, all_output=all_output)

        if do_record:
            if not delay_recording:
                if data_id is None:
                    if self._last_data_IDs:
                        for d_id in self._last_data_IDs.values():
                            self.fmkDB.insert_fmk_info(d_id, info, now)
                    else:
                        self.fmkDB.insert_fmk_info(None, info, now)
                else:
                    self.fmkDB.insert_fmk_info(data_id, info, now)
            else:
                self._current_fmk_info.append((info, now))

    def collect_feedback(self, content, status_code=None, subref=None, fbk_src=None):
        """
        Used within the scope of the Logger feedback-collector infrastructure.
        If your target implement the interface :meth:`Target.get_feedback`, no need to
        use this infrastructure.

        To be called by the target each time feedback need to be registered.

        Args:
            content: feedback record
            status_code (int): should be negative for error
            subref (str): specific reference to distinguish internal log sources within the same caller
            fbk_src: [optional] source object of the feedback
        """
        now = datetime.datetime.now()
        # TODO: get_caller_object() temporary blocks continuous_monitoring run within a thread
        # fbk_src = get_caller_object() if fbk_src is None else fbk_src

        with self._tg_fbk_lck:
            self._tg_fbk.append(
                (now, FeedbackSource(fbk_src, subref=subref), content, status_code)
            )

    def shall_record(self):
        if self.last_data_recordable or not self.__explicit_data_recording:
            return True
        else:
            # feedback will not be recorded because data is not recorded
            return False

    def _log_feedback(self, source, content, status_code, timestamp,
                      record=True, store_in_db=True, from_fbk_handler=False,
                      related_data_id=None):
        processed_feedback = self._process_target_feedback(content)
        fbk_cond = status_code is not None and status_code < 0
        hdr_color = Color.FEEDBACK_ERR if fbk_cond else Color.FEEDBACK
        body_color = Color.FEEDBACK_HLIGHT if fbk_cond else None
        # now = timestamp.strftime("%d/%m/%Y - %H:%M:%S.%f")
        if isinstance(timestamp, datetime.datetime) or timestamp is None:
            ts_msg = f"received at {timestamp}"
        elif isinstance(timestamp, list):
            if len(timestamp) == 1:
                ts_msg = f"received at {timestamp[0]}"
            else:
                ts_msg = f"received from {timestamp[0]} to {timestamp[-1]}"
        else:
            raise ValueError(f"Wrong format for timestamp [{type(timestamp)}]")

        if not processed_feedback:
            msg_hdr = "### Status from '{!s}': {!s} - {:s}".format(
                source, status_code, ts_msg
            )
        else:
            msg_hdr = "### Feedback from '{!s}' (status={!s}) - {:s}:".format(
                source, status_code, ts_msg
            )
        self.log_fn(msg_hdr, rgb=hdr_color, do_record=record)
        if processed_feedback:
            if source.display_feedback:
                if isinstance(processed_feedback, list):
                    for dfbk in processed_feedback:
                        self.log_fn(dfbk, rgb=body_color, do_record=record)
                else:
                    self.log_fn(processed_feedback, rgb=body_color, do_record=record)
            else:
                self.log_fn(
                    "Feedback not displayed", rgb=Color.WARNING, do_record=record
                )

        if isinstance(content, list):
            content = list(map(lambda x: self._encode_target_feedback(x), content))
        else:
            content = self._encode_target_feedback(content)

        if self.current_project and not from_fbk_handler:
            if not isinstance(source, FeedbackSource):
                source = FeedbackSource(source)
            self.current_project.trigger_feedback_handlers(source, timestamp, content, status_code,
                                                           self._returned_last_data_ID)

        if record:
            assert isinstance(source, FeedbackSource)
            if related_data_id is not None:
                data_id = related_data_id

            elif source.related_tg is not None:
                try:
                    data_id = self._last_data_IDs[source.related_tg]
                except KeyError:
                    self.print_console(
                        "*** Warning: The feedback source is related to a target to which nothing has been sent."
                        " Retrieved feedback will not be attached to any data ID.",
                        nl_before=True,
                        rgb=Color.WARNING,
                    )
                    data_id = None
            else:
                ids = self._last_data_IDs.values()
                data_id = max(ids) if ids else None

            if isinstance(content, (list, map)):
                for fbk, ts in zip(content, timestamp, strict=True):
                    self.fmkDB.insert_feedback(
                        data_id,
                        source,
                        ts,
                        fbk,
                        status_code=status_code,
                        store_in_db=store_in_db,
                        from_fb_handler=from_fbk_handler,
                    )
            else:
                self.fmkDB.insert_feedback(
                    data_id,
                    source,
                    timestamp,
                    content,
                    status_code=status_code,
                    store_in_db=store_in_db,
                    from_fb_handler=from_fbk_handler,
                )


    def log_collected_feedback(self, preamble=None, epilogue=None,
                               store_in_db=True):
        """
        Used within the scope of the Logger feedback-collector feature.
        If your target implement the interface :meth:`Target.get_feedback`, no need to
        use this infrastructure.

        It allows to retrieve the collected feedback, that has been populated
        by the target (through call to :meth:`Logger.collect_feedback`).

        Args:
            preamble (str): prefix added to each collected feedback
            epilogue (str): suffix added to each collected feedback

        Returns:
            bool: True if target feedback has been collected through logger infrastructure
              :meth:`Logger.collect_feedback`, False otherwise.
        """
        collected_status = {}

        with self._tg_fbk_lck:
            fbk_list = self._tg_fbk
            self._tg_fbk = []

        if not fbk_list:
            # self.log_fn("\n::[ NO TARGET FEEDBACK ]::\n")
            raise NotImplementedError

        record = self.shall_record()

        if preamble is not None:
            self.log_fn(preamble, do_record=record, rgb=Color.FMKINFO)

        for idx, fbk_record in enumerate(fbk_list):
            timestamp, fbk_src, fbk, status = fbk_record
            self._log_feedback(fbk_src, fbk, status, timestamp,
                               record=record, store_in_db=store_in_db)
            collected_status[fbk_src.obj] = status

        if epilogue is not None:
            self.log_fn(epilogue, do_record=record, rgb=Color.FMKINFO)

        return collected_status

    def log_target_feedback_from(
        self, source, content, status_code, timestamp, preamble=None, epilogue=None,
        store_in_db=True
    ):
        record = self.shall_record()

        if preamble is not None:
            self.log_fn(preamble, do_record=record, rgb=Color.FMKINFO)

        self._log_feedback(source, content, status_code, timestamp,
                           record=record, store_in_db=store_in_db)

        if epilogue is not None:
            self.log_fn(epilogue, do_record=record, rgb=Color.FMKINFO)


    def log_fbkhandler_feedback(self, fbkhandler, content, status_code, timestamp, store_in_db=True,
                                related_data_id=None):
        self._log_feedback(
            FeedbackSource(fbkhandler),
            content,
            status_code,
            timestamp,
            record=self.shall_record(),
            store_in_db=store_in_db,
            from_fbk_handler=True,
            related_data_id=related_data_id
        )


    def log_director_feedback(self, director, content, status_code, timestamp, store_in_db=True):
        self._log_feedback(
            FeedbackSource(director),
            content,
            status_code,
            timestamp,
            record=self.shall_record(),
            store_in_db=store_in_db
        )

    def log_probe_feedback(
        self, probe, content, status_code, timestamp, related_tg=None,
        store_in_db=True
    ):
        self._log_feedback(
            FeedbackSource(probe, related_tg=related_tg),
            content,
            status_code,
            timestamp,
            record=self.shall_record(),
            store_in_db=store_in_db
        )

    def _process_target_feedback(self, feedback):
        if feedback is None:
            return None

        if isinstance(feedback, list):
            new_fbk = []
            for f in feedback:
                if f is None:
                    continue
                new_f = f.strip()
                if isinstance(new_f, bytes):
                    new_f = self._handle_binary_content(new_f, raw=self.export_raw_data)
                new_fbk.append(new_f)

            if not new_fbk:
                new_fbk = None
            elif not list(filter(lambda x: x != b"", new_fbk)):
                new_fbk = None
        else:
            new_fbk = feedback.strip()
            if isinstance(new_fbk, bytes):
                new_fbk = self._handle_binary_content(new_fbk, raw=self.export_raw_data)

        return new_fbk

    def _encode_target_feedback(self, feedback):
        if feedback is None:
            return None
        return convert_to_internal_repr(feedback)

    PREAMBLE_PREFIX = '====[ Data Record #'

    def start_new_log_entry(self):
        self.__idx += 1
        self._current_sent_date = datetime.datetime.now()
        now = self._current_sent_date.strftime("%d/%m/%Y - %H:%M:%S.%f")
        msg = f"{self.PREAMBLE_PREFIX}{self.__idx} ]==[ {now} ]===="
        msg += "=" * (max(80 - len(msg), 0))
        self.log_fn(msg, rgb=Color.NEWLOGENTRY, style=FontStyle.BOLD, all_output=True)

        return self._current_sent_date

    EPILOGUE_PREFIX = '====[ End Record #'

    def stop_log_entry(self):
        msg = f'{self.EPILOGUE_PREFIX}{self.__idx} ]=='
        msg += "=" * (max(80 - len(msg), 0))
        self.log_fn(msg, rgb=Color.NEWLOGENTRY, style=FontStyle.BOLD, all_output=True)

    def log_dmaker_step(self, num):
        msg = "### Step %d:" % num
        self.log_fn(msg, rgb=Color.DMAKERSTEP, all_output=True)

    def log_generator_info(
        self, dmaker_type, name, user_input, data_id=None, disabled=False
    ):
        msg = "### Initial Generator (currently disabled):\n" if disabled else ""
        msg += "" if data_id is None else f" |- retrieved from data id: {data_id}\n"
        msg += f"|- generator type: {dmaker_type} | generator name: {name}"
        msg += f"\n|- user input: {user_input}" if user_input else f"\n|- no user input"
        msg += "\n  ..." if disabled else ""

        if not disabled:
            self._current_dmaker_list.append((dmaker_type, name, user_input))
            self._current_src_data_id = data_id
        self.log_fn(msg, rgb=Color.DISABLED if disabled else Color.DATAINFO, all_output=True)

    def log_operator_info(self, dmaker_type, name, user_input):
        msg = f"|- operator type: {dmaker_type} | operator name: {name}"
        msg += f"\n|- user input: {user_input}" if user_input else f"\n|- no user input"

        self._current_dmaker_list.append((dmaker_type, name, user_input))
        self.log_fn(msg, rgb=Color.DATAINFO, all_output=True)

    def log_data_info(self, data_info, dmaker_type, data_maker_name):
        if not data_info:
            return

        self._current_dmaker_info[(dmaker_type, data_maker_name)] = data_info

        self.log_fn("|- data info:", rgb=Color.DATAINFO, all_output=True)
        for msg in data_info:
            if len(msg) > self._term_display_limit:
                msg = msg[: self._term_display_limit] + " ..."

            self.log_fn("    | " + msg, rgb=Color.DATAINFO, all_output=True)

    def log_post_processed_info(self):
        if self._post_processed_info:
            for info in self._post_processed_info:
                self.log_fmk_info(info)

        self._post_processed_info = None

    def register_post_processed_info(self, info):
        if self._post_processed_info is None:
            self._post_processed_info = []
        self._post_processed_info.append(info)

    def log_info(self, info):
        msg = "### Info: {:s}".format(info)
        self.log_fn(msg, rgb=Color.INFO)

    def log_target_ack_date(self):
        for tg_ref, ack_date in self._current_ack_dates.items():
            msg = f"### Ack from '{tg_ref!s}' received at: "
            self.log_fn(msg, nl_after=False, rgb=Color.LOGSECTION, all_output=True)
            self.log_fn(str(ack_date), nl_before=False, all_output=True)

    def set_target_ack_date(self, tg_ref, date):
        if self._current_ack_dates is None:
            self._current_ack_dates = {tg_ref: date}
        else:
            self._current_ack_dates[tg_ref] = date

    def log_data(self, data, verbose=False):
        ret = True

        self.log_fn("### Data size: ", rgb=Color.LOGSECTION, nl_after=False, all_output=True)
        self._current_size = data.get_length()
        self.log_fn("%d bytes" % self._current_size, nl_before=False, all_output=True)

        if self.__explicit_data_recording and not data.is_recordable():
            self.last_data_recordable = False
            self.log_fn("### Data emitted but not recorded", rgb=Color.LOGSECTION, all_output=True)
            return False

        self._current_data = data
        self.last_data_recordable = self._current_data.is_recordable()

        if not self.__record_data:
            self.log_fn("### Data emitted:", rgb=Color.LOGSECTION, all_output=True)
            self.log_fn(data, nl_after=True, verbose=verbose, all_output=True)
        else:
            ffn = self._export_data_func(data)
            if ffn:
                self.log_fn(
                    "### Emitted data is stored in the file:", rgb=Color.LOGSECTION,
                    all_output=True
                )
                self.log_fn(ffn, all_output=True)
            else:
                self.print_console(
                    "ERROR: saving data in an external file has failed!",
                    nl_before=True,
                    rgb=Color.ERROR,
                    basic_output=True
                )
                ret = False

        return ret

    def _export_data_func(self, data, suffix=""):
        base_dir = gr.exported_data_folder

        dm = data.get_data_model()
        if dm:
            file_extension = dm.file_extension
            dm_name = dm.name
        else:
            file_extension = "bin"
            dm_name = "__unknown_data_model"

        now = datetime.datetime.now()
        current_export_date = now.strftime("%Y_%m_%d_%H%M%S")

        if current_export_date != self.__prev_export_date:
            self.__prev_export_date = current_export_date
            self.__export_cpt = 0
        else:
            self.__export_cpt += 1

        export_fname = "{date:s}_{cpt:0>2d}{suff:s}.{ext:s}".format(
            date=current_export_date,
            cpt=self.__export_cpt,
            ext=file_extension,
            suff=suffix,
        )

        export_full_fn = os.path.join(base_dir, dm_name, export_fname)

        gr.ensure_dir(export_full_fn)

        fd = open(export_full_fn, "wb")
        fd.write(data.to_bytes())
        fd.close()

        return export_full_fn

    def log_comment(self, comment):
        now = datetime.datetime.now()
        current_date = now.strftime("%H:%M:%S")

        self.log_fn(
            "### Comments [{date:s}]:".format(date=current_date), rgb=Color.COMMENTS
        )
        self.log_fn(comment)
        data_id_list = self._last_data_IDs.values()
        data_id = max(data_id_list) if data_id_list else None
        self.fmkDB.insert_comment(data_id, comment, now)
        self.print_console("\n")

    def log_error(self, err_msg):
        now = datetime.datetime.now()
        msg = "\n/!\\ ERROR: %s /!\\\n" % err_msg
        self.log_fn(msg, rgb=Color.ERROR)
        data_id_list = self._last_data_IDs.values()
        data_id = max(data_id_list) if data_id_list else None
        self.fmkDB.insert_fmk_info(data_id, msg, now, error=True)

    def print_console(
            self,
            msg,
            nl_before=True,
            nl_after=False,
            rgb=None,
            style=None,
            raw_limit=None,
            limit_output=True,
            no_format_mode=False,
            basic_output=False,
            all_output=False,
    ):
        with self._sync_lock:
            with self._log_entry_submitted_cond:
                params = (
                    msg,
                    nl_before,
                    nl_after,
                    rgb,
                    style,
                    raw_limit,
                    limit_output,
                    no_format_mode,
                    basic_output,
                    all_output
                )
                self._log_entry_list.append((Logger.PRINT_CONSOLE_API, params))
                self._log_entry_submitted_cond.notify()

    def _print_console(
            self,
            msg,
            nl_before=True,
            nl_after=False,
            rgb=None,
            style=None,
            raw_limit=None,
            limit_output=True,
            no_format_mode=False,
            basic_output=False,
            all_output = False
    ):
        if not self.display_on_term:
            return

        if raw_limit is None:
            raw_limit = self._term_display_limit

        p = "\n" if nl_before else ""
        s = "\n" if nl_after else ""

        prefix = p + self.p

        if no_format_mode:
            if self._ext_disp.is_enabled:
                if all_output or basic_output:
                    self._ext_disp.disp.print_main(prefix + msg, newline=False)
                if all_output or not basic_output:
                    self._ext_disp.disp.print(prefix + msg, newline=False)
            else:
                sys.stdout.write(prefix + msg)
                sys.stdout.flush()

        else:
            if isinstance(msg, Data):
                msg = repr(msg)

            suffix = ""
            if limit_output and len(msg) > raw_limit:
                msg = msg[:raw_limit]
                suffix = " ..."

            suffix += s

            # if self._tui:
            #     msg = msg.replace('[', r'\[')
            #     if rgb is None:
            #         color = ''
            #     else:
            #         color = Color.to_bbcode(rgb)
            # else:
            if rgb is not None:
                msg = colorize(msg, rgb=rgb)

            if style is None:
                style = ''

            if self._ext_disp.is_enabled:
                # if self._tui:
                #     self._ext_disp.disp.set_basic_output_mode(markup=True)
                #     time.sleep(0.1)
                #     self._ext_disp.disp.print_main(
                #         '[' + self.style2bbcode[style] + color + ']' + prefix + msg + suffix + '[/]')
                #     self._ext_disp.disp.set_basic_output_mode(markup=False)
                # else:
                if all_output or basic_output:
                    self._ext_disp.disp.print_main(style + prefix + msg + suffix + FontStyle.END,
                                                   newline=False)
                if all_output or not basic_output:
                    self._ext_disp.disp.print(prefix + msg, newline=False)
            else:
                sys.stdout.write(style + prefix + msg + suffix + FontStyle.END)
                sys.stdout.flush()
