/*
 * This file is part of the Code::Blocks IDE and licensed under the GNU Lesser General Public License, version 3
 * http://www.gnu.org/licenses/lgpl-3.0.html
 *
 * $Revision: 13101 $
 * $Id: encodingdetector.cpp 13101 2022-12-08 18:11:23Z wh11204 $
 * $HeadURL: https://svn.code.sf.net/p/codeblocks/code/trunk/src/sdk/encodingdetector.cpp $
 */

#include "sdk_precomp.h"
#ifndef CB_PRECOMP
    #include <wx/fontmap.h>
    #include <wx/file.h>
    #include <wx/string.h>
    #include "manager.h"
    #include "logmanager.h"
    #include "configmanager.h"
#endif // CB_PRECOMP


#include "encodingdetector.h"
#include "filemanager.h"

#include "nsError.h"
#include "nsUniversalDetector.h"

#include <wx/encconv.h>

/* ----------------------------------------------
 *  Some detection code is borrowed from MadEdit,
 *  but modified to suit C::B. Other portions are
 *  using the Mozilla universal char detector.
 * ---------------------------------------------- */

/// Convert the char buffer to wxString and if there are any null-terminating characters at the end - remove them.
inline wxString makeStringNoNull(const wxWCharBuffer &wideBuff)
{
    wxString result(wideBuff);
    if (!result.empty())
    {
        wxString::size_type ii = result.find_last_not_of(wxT('\0'));
        if (ii != wxString::npos)
            result.resize(ii + 1);
    }
    return result;
}

class EncodingDetectorImpl : public nsUniversalDetector
{
    public:
        EncodingDetectorImpl(bool useLog) :
            nsUniversalDetector(NS_FILTER_ALL),
            m_IsOK(false),
            m_UseBOM(false),
            m_UseLog(useLog),
            m_BOMSizeInBytes(0),
            m_ConvStr(wxEmptyString)
        {
            m_Encoding = wxLocale::GetSystemEncoding();
        }

        /** @return True if succeeded, false if not (e.g. file didn't exist). */
        bool DetectEncoding(const wxString& filename)
        {
            wxFile file(filename);
            if (!file.IsOpened())
                return false;

            size_t size = file.Length();
            if (size == 0)
            {
                file.Close();
                return false;
            }

            wxByte* buffer = (wxByte*) malloc(sizeof(wxByte) * (size + 4));
            if (!buffer)
            {
                file.Close();
                return false;
            }
            buffer[size + 0] = 0;
            buffer[size + 1] = 0;
            buffer[size + 2] = 0;
            buffer[size + 3] = 0;

            size_t readBytes = file.Read((void*)buffer, size);
            bool result = false;
            if (readBytes > 0)
                result = DetectEncoding(buffer, size);

            file.Close();
            free(buffer);
            return result;
        }

        bool DetectEncoding(const wxByte* buffer, size_t size)
        {
            ConfigManager* cfgMgr = Manager::Get()->GetConfigManager(_T("editor"));
            const wxString &encname = cfgMgr->Read(_T("/default_encoding"),
                                                   wxLocale::GetSystemEncodingName());

            if (cfgMgr->ReadInt(_T("/default_encoding/use_option"), 0) == 1)
            {
                // Bypass C::B's auto-detection
                m_Encoding = wxFontMapper::Get()->CharsetToEncoding(encname, false);

                if (m_UseLog)
                {
                    wxString msg;
                    msg.Printf(_T("Warning: bypassing C::B's auto-detection!\n"
                                  "Encoding requested is: %s (ID: %d)"),
                               wxFontMapper::Get()->GetEncodingDescription(m_Encoding).c_str(),
                               m_Encoding);
                    Manager::Get()->GetLogManager()->DebugLog(msg);
                }
            }
            else
            {
                if (!buffer)
                    return false;

                // Try our own detection for UTF-16 and UTF-32, the Mozilla-version does not work without BOM
                if ( DetectEncodingEx(buffer, size) )
                {
                    if (m_UseBOM && m_UseLog)
                    {
                        wxString msg;
                        msg.Printf(_T("Detected encoding via BOM: %s (ID: %d)"),
                                   wxFontMapper::Get()->GetEncodingDescription(m_Encoding).c_str(),
                                   m_Encoding);
                        Manager::Get()->GetLogManager()->DebugLog(msg);
                    }
                }
                else
                {
                    //{ MOZILLA nsUniversalDetector START
                    // If we still have no results try Mozilla (taken from nsUdetXPCOMWrapper.cpp):
                    Reset();
                    nsresult res = HandleData((char*)buffer, size);
                    if (res==NS_OK)
                        DataEnd();
                    else
                    {
                        m_MozillaResult = wxEmptyString;
                        if (m_UseLog)
                            Manager::Get()->GetLogManager()->DebugLog(wxString::Format("Mozilla universal detection failed with %d.", res));
                    }
                    //} MOZILLA nsUniversalDetector END

                    if ( !m_MozillaResult.IsEmpty() )
                        m_Encoding = wxFontMapper::Get()->CharsetToEncoding(m_MozillaResult, false);

                    if (m_Encoding == wxFONTENCODING_DEFAULT)
                    {
                        wxString enc_name = Manager::Get()->GetConfigManager(_T("editor"))->Read(_T("/default_encoding"), wxLocale::GetSystemEncodingName());
                        m_Encoding = wxFontMapper::GetEncodingFromName(enc_name);
                        if (m_UseLog)
                        {
                            wxString msg;
                            msg.Printf(_T("Text seems to be pure ASCII!\n"
                                          "We use user specified encoding: %s (ID: %d)"),
                                       wxFontMapper::Get()->GetEncodingDescription(m_Encoding).c_str(),
                                       m_Encoding);
                            Manager::Get()->GetLogManager()->DebugLog(msg);
                        }
                    }

                    if (m_Encoding < 0)
                    {
                        // Use user-specified one; as a fallback
                        m_Encoding = wxFontMapper::Get()->CharsetToEncoding(encname, false);
                        if (m_UseLog)
                        {
                            wxString msg;
                            msg.Printf(_T("Warning: Using user specified encoding as fallback!\n"
                                          "Encoding fallback is: %s (ID: %d)"),
                                       wxFontMapper::Get()->GetEncodingDescription(m_Encoding).c_str(),
                                       m_Encoding);
                            Manager::Get()->GetLogManager()->DebugLog(msg);
                        }
                    }

                    m_UseBOM = false;
                    m_BOMSizeInBytes = 0;
                }
            }

            if (m_UseLog)
            {
                wxString msg;
                msg.Printf(_T("Final encoding detected: %s (ID: %d)"),
                           wxFontMapper::Get()->GetEncodingDescription(m_Encoding).c_str(),
                           m_Encoding);
                Manager::Get()->GetLogManager()->DebugLog(msg);
            }

            if (!ConvertToWxString(buffer, size) && m_UseLog)
                Manager::Get()->GetLogManager()->DebugLog(_T("Something seriously went wrong while converting file content to wxString!"));

            return true;
        }

        // Stolen from  https://github.com/etexteditor/e/blob/master/src/Strings.cpp
        //        and:  https://github.com/etexteditor/e/blob/master/src/Utf.cpp
        // Copyright (c) 2009, Alexander Stigsen, e-texteditor.com (All rights reserved)
        // http://www.e-texteditor.com/
        bool DetectEncodingEx(const wxByte* buffer, size_t size)
        {
            if (!buffer || size == 0)
                return false;

            const wxByte*  buff_ptr = buffer;
            const wxByte*  buff_end = &buffer[size];
            wxFontEncoding enc      = wxFONTENCODING_DEFAULT;

            // Check if the buffer starts with a BOM (Byte Order Marker)
            if (size >= 2)
            {
                if      (size >= 4 && memcmp(buffer, "\xFF\xFE\x00\x00", 4) == 0)
                {
                    enc = wxFONTENCODING_UTF32LE;
                    m_BOMSizeInBytes = 4;
                    m_UseBOM = true;
                }
                else if (size >= 4 && memcmp(buffer, "\xFE\xFF\x00\x00", 4) == 0)
                {
                    // FE FF 00 00  UCS-4, unusual octet order BOM (3412)
                    // X-ISO-10646-UCS-4-3412 can not (yet) be handled by wxWidgets
                    enc = (wxFontEncoding)-1;
                }
                else if (size >= 4 && memcmp(buffer, "\x00\x00\xFE\xFF", 4) == 0)
                {
                    enc = wxFONTENCODING_UTF32BE;
                    m_BOMSizeInBytes = 4;
                    m_UseBOM = true;
                }
                else if (size >= 4 && memcmp(buffer, "\x00\x00\xFF\xFE", 4) == 0)
                {
                    // 00 00 FF FE  UCS-4, unusual octet order BOM (2143)
                    // X-ISO-10646-UCS-4-2143 can not (yet) be handled by wxWidgets
                    enc = (wxFontEncoding)-1;
                }
                else if (             memcmp(buffer, "\xFF\xFE", 2) == 0)
                {
                    enc = wxFONTENCODING_UTF16LE;
                    m_BOMSizeInBytes = 2;
                    m_UseBOM = true;
                }
                else if (             memcmp(buffer, "\xFE\xFF", 2) == 0)
                {
                    enc = wxFONTENCODING_UTF16BE;
                    m_BOMSizeInBytes = 2;
                    m_UseBOM = true;
                }
                else if (size >= 3 && memcmp(buffer, "\xEF\xBB\xBF", 3) == 0)
                {
                    enc = wxFONTENCODING_UTF8;
                    m_BOMSizeInBytes = 3;
                    m_UseBOM = true;
                }
                else if (size >= 5 && memcmp(buffer, "\x2B\x2F\x76\x38\x2D", 5) == 0)
                {
                    enc = wxFONTENCODING_UTF7;
                    m_BOMSizeInBytes = 5;
                    m_UseBOM = true;
                }

                buff_ptr += m_BOMSizeInBytes;
            }

            // If the file starts with a leading < (less) sign, it is probably an XML file
            // and we can determine the encoding by how the sign is encoded.
            if (enc == wxFONTENCODING_DEFAULT && size >= 2)
            {
                if      (size >= 4 && memcmp(buffer, "\x3C\x00\x00\x00", 4) == 0) enc = wxFONTENCODING_UTF32LE;
                else if (size >= 4 && memcmp(buffer, "\x00\x00\x00\x3C", 4) == 0) enc = wxFONTENCODING_UTF32BE;
                else if (             memcmp(buffer, "\x3C\x00",         2) == 0) enc = wxFONTENCODING_UTF16LE;
                else if (             memcmp(buffer, "\x00\x3C",         2) == 0) enc = wxFONTENCODING_UTF16BE;
            }

            // Unicode Detection
            if (enc == wxFONTENCODING_DEFAULT)
            {
                unsigned int null_byte_count  = 0;
                unsigned int utf_bytes        = 0;
                unsigned int good_utf_count   = 0;
                unsigned int bad_utf_count    = 0;
                unsigned int bad_utf32_count  = 0;
                unsigned int bad_utf16_count  = 0;
                unsigned int nl_utf32le_count = 0;
                unsigned int nl_utf32be_count = 0;
                unsigned int nl_utf16le_count = 0;
                unsigned int nl_utf16be_count = 0;

                while (buff_ptr != buff_end)
                {
                    if (*buff_ptr == 0) ++null_byte_count;

                    // Detect UTF-8 by scanning for invalid sequences
                    if (utf_bytes == 0)
                    {
                        if ((*buff_ptr & 0xC0) == 0x80 || *buff_ptr == 0)
                            ++bad_utf_count;
                        else
                        {
                            const char c = *buff_ptr;
                            utf_bytes = 5; // invalid length
                            if      ((c & 0x80) == 0x00) utf_bytes = 1;
                            else if ((c & 0xE0) == 0xC0) utf_bytes = 2;
                            else if ((c & 0xF0) == 0xE0) utf_bytes = 3;
                            else if ((c & 0xF8) == 0xF0) utf_bytes = 4;
                            if (utf_bytes > 3)
                            {
                                ++bad_utf_count;
                                utf_bytes = 0;
                            }
                        }
                    }
                    else if ((*buff_ptr & 0xC0) == 0x80)
                    {
                        --utf_bytes;
                        if (utf_bytes == 0)
                            ++good_utf_count;
                    }
                    else
                    {
                        ++bad_utf_count;
                        utf_bytes = 0;
                    }

                    // Detect UTF-32 by scanning for newlines (and lack of null chars)
                    if ((wxUIntPtr)buff_ptr % 4 == 0 && buff_ptr+4 <= buff_end)
                    {
                        if (*((wxUint32*)buff_ptr) == 0                        ) ++bad_utf32_count;
                        if (*((wxUint32*)buff_ptr) == wxUINT32_SWAP_ON_BE(0x0A)) ++nl_utf32le_count;
                        if (*((wxUint32*)buff_ptr) == wxUINT32_SWAP_ON_LE(0x0A)) ++nl_utf32be_count;
                    }

                    // Detect UTF-16 by scanning for newlines (and lack of null chars)
                    if ((wxUIntPtr)buff_ptr % 2 == 0 && buff_ptr+4 <= buff_end)
                    {
                        if (*((wxUint16*)buff_ptr) == 0)                         ++bad_utf16_count;
                        if (*((wxUint16*)buff_ptr) == wxUINT16_SWAP_ON_BE(0x0A)) ++nl_utf16le_count;
                        if (*((wxUint16*)buff_ptr) == wxUINT16_SWAP_ON_LE(0x0A)) ++nl_utf16be_count;
                    }

                    ++buff_ptr;
                }

                if      (bad_utf_count   == 0)                                  enc = wxFONTENCODING_UTF8;
                else if (bad_utf32_count == 0 && nl_utf32le_count > size / 400) enc = wxFONTENCODING_UTF32LE;
                else if (bad_utf32_count == 0 && nl_utf32be_count > size / 400) enc = wxFONTENCODING_UTF32BE;
                else if (bad_utf16_count == 0 && nl_utf16le_count > size / 200) enc = wxFONTENCODING_UTF16LE;
                else if (bad_utf16_count == 0 && nl_utf16be_count > size / 200) enc = wxFONTENCODING_UTF16BE;
                else if (null_byte_count)
                    return false; // Maybe this is a binary file?
            }

            if (enc != wxFONTENCODING_DEFAULT)
            {
                m_Encoding = enc; // Success.
                return true;
            }

            // If we can't detect encoding and it does not contain null bytes
            // just ignore it and try backup-procedures (Mozilla) later...
            return false;

        }

        void Report(const char* aCharset) override
        {
            m_MozillaResult = cbC2U(aCharset);

            if (m_UseLog)
                Manager::Get()->GetLogManager()->DebugLog(wxString::Format("Mozilla universal detection engine detected '%s'.", m_MozillaResult));

            if (m_MozillaResult == _T("gb18030")) // hack, because wxWidgets only knows cp936
                m_MozillaResult = _T("cp936");
            else if (m_MozillaResult.Contains(wxT("*ASCII*"))) // remove our "specials"
                m_MozillaResult = wxEmptyString;
        }

        bool           m_IsOK;
        bool           m_UseBOM;
        bool           m_UseLog;
        int            m_BOMSizeInBytes;
        wxFontEncoding m_Encoding;

        bool ConvertToWxString(const wxByte* buffer, size_t size)
        {
            LogManager* logmgr = Manager::Get()->GetLogManager();
            wxString    logmsg;

            if (!buffer || size == 0)
            {
                if (m_UseLog)
                {
                    logmsg.Printf(_T("Encoding conversion has failed (buffer is empty)!"));
                    logmgr->DebugLog(logmsg);
                }
                return false; // Nothing we can do...
            }

            if (m_BOMSizeInBytes > 0)
            {
                for (int i = 0; i < m_BOMSizeInBytes; ++i)
                    buffer++;
            }

            size_t outlen = 0;

            /* NOTE (Biplab#5#): FileManager returns a buffer with 4 extra NULL chars appended.
               But the buffer size is returned sans the NULL chars */

            wxWCharBuffer wideBuff;

            // if possible use the special conversion-routines, they are much faster than wxCSCov (at least on linux)
            if      ( m_Encoding == wxFONTENCODING_UTF7 )
            {
                wxMBConvUTF7 conv;
                wideBuff = conv.cMB2WC((const char*)buffer, size + 4 - m_BOMSizeInBytes, &outlen);
            }
            else if ( m_Encoding == wxFONTENCODING_UTF8 )
            {
                wxMBConvUTF8 conv;
                wideBuff = conv.cMB2WC((const char*)buffer, size + 4 - m_BOMSizeInBytes, &outlen);
            }
            else if ( m_Encoding == wxFONTENCODING_UTF16BE )
            {
                wxMBConvUTF16BE conv;
                wideBuff = conv.cMB2WC((const char*)buffer, size + 4 - m_BOMSizeInBytes, &outlen);
            }
            else if ( m_Encoding == wxFONTENCODING_UTF16LE )
            {
                wxMBConvUTF16LE conv;
                wideBuff = conv.cMB2WC((const char*)buffer, size + 4 - m_BOMSizeInBytes, &outlen);
            }
            else if ( m_Encoding == wxFONTENCODING_UTF32BE )
            {
                wxMBConvUTF32BE conv;
                wideBuff = conv.cMB2WC((const char*)buffer, size + 4 - m_BOMSizeInBytes, &outlen);
            }
            else if ( m_Encoding == wxFONTENCODING_UTF32LE )
            {
                wxMBConvUTF32LE conv;
                wideBuff = conv.cMB2WC((const char*)buffer, size + 4 - m_BOMSizeInBytes, &outlen);
            }
            else
            {
                // try wxEncodingConverter first, even it it only works for
                // wxFONTENCODING_ISO8859_1..15, wxFONTENCODING_CP1250..1257 and wxFONTENCODING_KOI8
                // but it's much, much faster than wxCSConv (at least on Linux)
                wxEncodingConverter conv;
                wchar_t* tmp = new wchar_t[size + 4 - m_BOMSizeInBytes];
                if (  conv.Init(m_Encoding, wxFONTENCODING_UNICODE)
                   && conv.Convert((const char*)buffer, tmp) )
                {
                    wideBuff = tmp;
                    outlen = size + 4 - m_BOMSizeInBytes; // should be correct, because Convert has returned true
                    if (m_UseLog && outlen>0)
                    {
                        logmsg.Printf("Conversion succeeded using wxEncodingConverter "
                                      "(buffer size = %zu, converted size = %zu.", size, outlen);
                        logmgr->DebugLog(logmsg);
                    }
                }
                else
                {
                    // try wxCSConv, if nothing else works
                    wxCSConv csconv(m_Encoding);
                    if (csconv.IsOk())
                    {
                        wideBuff = csconv.cMB2WC((const char*)buffer, size + 4 - m_BOMSizeInBytes, &outlen);
                        if (m_UseLog && outlen>0)
                        {
                            logmsg.Printf("Conversion succeeded using wxCSConv "
                                          "(buffer size = %zu, converted size = %zu.", size, outlen);
                            logmgr->DebugLog(logmsg);
                        }
                    }
                }
                delete [] tmp;
            }

            if (outlen>0)
            {
                m_ConvStr = makeStringNoNull(wideBuff);
                return true; // Done.
            }

            // Here, outlen == 0, so an error occurred during conversion.
            if (m_UseLog)
            {
                logmsg.Printf(_T("Encoding conversion using settings has failed!\n"
                                 "Encoding chosen was: %s (ID: %d)"),
                              wxFontMapper::Get()->GetEncodingDescription(m_Encoding).wx_str(),
                              m_Encoding);
                logmgr->DebugLog(logmsg);
            }

            // Try system locale as fall-back (if requested by the settings)
            ConfigManager* cfgMgr = Manager::Get()->GetConfigManager(_T("editor"));
            if (cfgMgr->ReadBool(_T("/default_encoding/use_system"), true))
            {
                if (platform::windows)
                {
                    if (m_UseLog)
                        logmgr->DebugLog(_T("Trying system locale as fallback..."));

                    m_Encoding = wxLocale::GetSystemEncoding();
                }
                else
                {
                    // We can rely on the UTF-8 detection code ;-)
                    if (m_UseLog)
                        logmgr->DebugLog(_T("Trying ISO-8859-1 as fallback..."));

                    m_Encoding = wxFONTENCODING_ISO8859_1;
                }

                wxCSConv conv_system(m_Encoding);
                wideBuff = conv_system.cMB2WC((const char*)buffer, size + 4 - m_BOMSizeInBytes, &outlen);
                m_ConvStr = makeStringNoNull(wideBuff);

                if (outlen == 0)
                {
                    if (m_UseLog)
                    {
                        logmsg.Printf(_T("Encoding conversion using system locale fallback has failed!\n"
                                         "Last encoding choosen was: %s (ID: %d)\n"
                                         "Don't know what to do."),
                                      wxFontMapper::Get()->GetEncodingDescription(m_Encoding).c_str(),
                                      m_Encoding);
                        logmgr->DebugLog(logmsg);
                    }
                    return false; // Nothing we can do...
                }
            }
            else
            {
                if (m_UseLog)
                {
                    logmgr->DebugLog(_T("Encoding conversion has seriously failed!\n"
                                        "Don't know what to do."));
                }
                return false; // Nothing we can do...
            }

            return true;
        }

        wxString m_MozillaResult;
        wxString m_ConvStr;
};

EncodingDetector::EncodingDetector(const wxString& filename, bool useLog)
{
    EncodingDetectorImpl detector(useLog);
    m_IsOK = detector.DetectEncoding(filename);
    m_ConvStr = detector.m_ConvStr;
    m_Encoding = detector.m_Encoding;
    m_BOMSizeInBytes = detector.m_BOMSizeInBytes;
    m_UseBOM = detector.m_UseBOM;
}

EncodingDetector::EncodingDetector(LoaderBase* fileLdr, bool useLog)
{
    EncodingDetectorImpl detector(useLog);
    m_IsOK = detector.DetectEncoding((wxByte*)fileLdr->GetData(), fileLdr->GetLength());
    m_ConvStr = detector.m_ConvStr;
    m_Encoding = detector.m_Encoding;
    m_BOMSizeInBytes = detector.m_BOMSizeInBytes;
    m_UseBOM = detector.m_UseBOM;
}

EncodingDetector::EncodingDetector(const wxByte* buffer, size_t size, bool useLog)
{
    EncodingDetectorImpl detector(useLog);
    m_IsOK = detector.DetectEncoding(buffer, size);
    m_ConvStr = detector.m_ConvStr;
    m_Encoding = detector.m_Encoding;
    m_BOMSizeInBytes = detector.m_BOMSizeInBytes;
    m_UseBOM = detector.m_UseBOM;
}

EncodingDetector::~EncodingDetector()
{
}

bool EncodingDetector::IsOK() const
{
    return m_IsOK;
}

bool EncodingDetector::UsesBOM() const
{
    return m_UseBOM;
}

int EncodingDetector::GetBOMSizeInBytes() const
{
    return m_BOMSizeInBytes;
}

wxFontEncoding EncodingDetector::GetFontEncoding() const
{
    return m_Encoding;
}

wxString EncodingDetector::GetWxStr() const
{
    return m_ConvStr;
}
