Make renew working well

Change po2lo output: write out the untranslated
sdf line too, write out translated sdf line only
if there is translation to the komponent, write out
fuzzy entries and indicate which entries are fuzzy.
Change GenPoEntry to write out fuzzy flag
Change renewpo to write out all type of
entries not just text and to remove id
duplicates which are made by xrmex

Change-Id: Idc9b6062638d1d5fcbbae75e0396996b63e9a7a0
Reviewed-on: https://gerrit.libreoffice.org/508Reviewed-by: Andras Timar <atimar@suse.com>
Tested-by: Andras Timar <atimar@suse.com>

l10ntools/inc/po.hxx

@@ -43,6 +43,7 @@ private:
     OString    m_sContext;
     OString    m_sUnTransStr;
     OString    m_sTransStr;
+    bool       m_bFuzzy;
 
 public:
                         GenPoEntry();
@@ -60,6 +61,8 @@ public:
                         { m_sUnTransStr = rUnTransStr; }
     virtual void        setTransStr(const OString& rTransStr)
                         { m_sTransStr = rTransStr; }
+    virtual void        setFuzzy(bool bFuzzy)
+                        { m_bFuzzy = bFuzzy; }
 
     virtual void        writeToFile(std::ofstream& io_rOFStream);
 };

l10ntools/scripts/po2lo

@@ -41,6 +41,7 @@ class Entry:
     """Represents a single line in an SDF file."""
 
     def __init__(self, items):
+        self.has_translation = None;
         self.items = items # list of 15 fields
         path = self.items[1].split('\\')
         self.po = "%s/%s/%s.po" % (options.input.replace('\\', '/'), self.items[0], "/".join(path[:-1]))
@@ -65,13 +66,20 @@ class Entry:
         """Translates text in the entry based on translations."""
 
         self.items[9] = options.language
+        self.items[2] = ""
+        self.has_translation = False
         for idx, key in self.keys:
             try:
-                self.items[idx] = translations.data[(self.po, key)]
+                if translations.data[(self.po, key)][1]:
+                    self.items[2] += "1"
+                else:
+                    self.items[2] += "0"
+                self.items[idx] = translations.data[(self.po, key)][0]
+                self.has_translation = True
 
                 self.items[14] = "2002-02-02 02:02:02"
             except KeyError:
-                pass
+                self.items[idx]=""
         self.items[14] = self.items[14].strip()
 
     def sdf2po(self, s):
@@ -95,8 +103,10 @@ class Template:
 
         sock = xopen(options.output, "w", encoding='utf-8')
         for line in self.lines:
+            sock.write("\t".join(line.items))
             line.translate(translations)
-            sock.write("\t".join(line.items)+"\r\n")
+            if line.has_translation:
+                sock.write("\t".join(line.items)+"\r\n")
         sock.close()
 
 class Translations:
@@ -120,10 +130,10 @@ class Translations:
                     elif line.startswith("msgstr "):
                         trans = line.strip()[8:-1]
                         if len(trans):
+                            self.setdata(path, key, trans, fuzzy)
                             if fuzzy:
                                 fuzzy = False
                             else:
-                                self.setdata(path, key, trans)
                                 multiline = False
                         else:
                             buf = []
@@ -132,16 +142,15 @@ class Translations:
                     elif multiline and line.startswith('"'):
                         buf.append(line.strip()[1:-1])
                     elif multiline and not len(line.strip()) and len("".join(buf)):
+                        self.setdata(path, key, "".join(buf),fuzzy)
                         if fuzzy:
                             fuzzy = False
-                        else:
-                            self.setdata(path, key, "".join(buf))
                         buf = []
                         multiline = False
-                if multiline and len("".join(buf)) and not fuzzy:
-                    self.setdata(path, key, "".join(buf))
+                if multiline and len("".join(buf)):
+                    self.setdata(path, key, "".join(buf),fuzzy)
 
-    def setdata(self, path, key, s):
+    def setdata(self, path, key, s, fuzzy = False):
         """Sets the translation for a given path and key, handling (un)escaping
         as well."""
         if key:
@@ -151,7 +160,7 @@ class Translations:
                 s = self.escape_help_text(s)
             else:
                 s = s.replace('\\\\', '\\')
-            self.data[(path.replace('\\', '/'), key)] = s
+            self.data[(path.replace('\\', '/'), key)] = ( s , fuzzy )
 
     def escape_help_text(self, text):
         """Escapes the help text as it would be in an SDF file."""

l10ntools/source/localize.cxx

@@ -281,15 +281,13 @@ void handleCommand(
         while (!in.eof())
         {
             OString sLine = OString(s.data(),s.length());
-            if (!sLine.isEmpty())
-            {
-                if (!sLine.getToken(PoEntry::TEXT,'\t').isEmpty())
-                    PoEntry(sLine).writeToFile(outPut);
-                if (!sLine.getToken(PoEntry::QUICKHELPTEXT,'\t').isEmpty())
-                    PoEntry(sLine,PoEntry::TQUICKHELPTEXT).writeToFile(outPut);
-                if (!sLine.getToken(PoEntry::TITLE,'\t').isEmpty())
-                    PoEntry(sLine,PoEntry::TTITLE).writeToFile(outPut);
-            }
+
+            if (!sLine.getToken(PoEntry::TEXT,'\t').isEmpty())
+                PoEntry(sLine).writeToFile(outPut);
+            if (!sLine.getToken(PoEntry::QUICKHELPTEXT,'\t').isEmpty())
+                PoEntry(sLine,PoEntry::TQUICKHELPTEXT).writeToFile(outPut);
+            if (!sLine.getToken(PoEntry::TITLE,'\t').isEmpty())
+                PoEntry(sLine,PoEntry::TTITLE).writeToFile(outPut);
             std::getline(in, s);
         };
         in.close();

l10ntools/source/po.cxx

@@ -100,6 +100,7 @@ GenPoEntry::GenPoEntry()
     , m_sContext( OString() )
     , m_sUnTransStr( OString() )
     , m_sTransStr( OString() )
+    , m_bFuzzy( false )
 {
 }
 
@@ -119,6 +120,8 @@ void GenPoEntry::writeToFile(std::ofstream& io_rOFStream)
                      << std::endl;
     if ( !m_sReference.isEmpty() )
         io_rOFStream << "#: " << m_sReference.getStr() << std::endl;
+    if ( m_bFuzzy )
+        io_rOFStream << "#, fuzzy" << std::endl;
     if ( !m_sContext.isEmpty() )
         io_rOFStream << "msgctxt "
                      << ImplGenMsgString(m_sContext).getStr() << std::endl;

l10ntools/source/renewpo.cxx

@@ -30,6 +30,7 @@
 #include <fstream>
 #include <dirent.h>
 #include <string>
+#include <vector>
 #include <rtl/string.hxx>
 #include "po.hxx"
 
@@ -40,7 +41,8 @@ bool IsSameEntry(const OString& rFirstEntry,const OString& rSecEntry)
 {
     for(int i = PoEntry::PROJECT; i<=PoEntry::LOCALID;++i)
     {
-        if ( rFirstEntry.getToken(i,'\t') != rSecEntry.getToken(i,'\t'))
+        if ( rFirstEntry.getToken(i,'\t') != rSecEntry.getToken(i,'\t') &&
+             i != PoEntry::DUMMY)
             return false;
     }
     return true;
@@ -57,19 +59,34 @@ OString GetPath(const OString& rPath, const OString& rLine)
     return sSourcePath;
 }
 
+OString DelLocalId(const OString& rLine)
+{
+    sal_uInt16 nTabIndex = 0;
+    for(sal_uInt16 nComponent=0; nComponent<PoEntry::LOCALID; ++nComponent)
+    {
+        nTabIndex = rLine.indexOf('\t',nTabIndex);
+        ++nTabIndex;
+    }
+    return rLine.replaceAt(nTabIndex,
+                           rLine.indexOf('\t',nTabIndex)-nTabIndex,
+                           "");
+}
+
 //Renew po files of the actual language
 void HandleLanguage(struct dirent* pLangEntry, const OString& rPath,
-                    const OString& rpo2ooPath, const OString& rSDFPath)
+                    const OString& rpo2loPath, const OString& rSDFPath)
 {
     const OString LangEntryName = pLangEntry->d_name;
     const OString SDFFileName = LangEntryName + ".sdf";
 
     //Generate and open sdf
-    cout << "Start process with language: " <<  LangEntryName.getStr() << endl;
-    system( (rpo2ooPath + " -t " + rSDFPath +
-            " -l " + LangEntryName + " " +
-            rPath.getStr() + LangEntryName + " " +
-            SDFFileName).getStr());
+    cout << "Process start with language: " <<  LangEntryName.getStr() << endl;
+    system( (rpo2loPath +
+            " -i " + rPath.getStr() + LangEntryName +
+            " -o " + SDFFileName +
+            " -l " + LangEntryName +
+            " -t " + rSDFPath).getStr());
+    cout << "Language sdf is ready!" << endl;
 
     ofstream aOutPut;
     ifstream aSDFInput(SDFFileName.getStr());
@@ -118,9 +135,31 @@ void HandleLanguage(struct dirent* pLangEntry, const OString& rPath,
         {
             sActTrans ="";
         }
-        PoEntry aPE(sActUnTrans);
-        aPE.setTransStr(sActTrans.getToken(PoEntry::TEXT,'\t'));
-        aPE.writeToFile(aOutPut);
+        const vector<PoEntry::TYPE> vTypes = { PoEntry::TTEXT,
+                                               PoEntry::TQUICKHELPTEXT,
+                                               PoEntry::TTITLE };
+        sal_uInt16 nDummyBit = 0;
+        for( sal_uInt16 nIndex=0; nIndex<vTypes.size(); ++nIndex)
+        {
+            if (!sActUnTrans.getToken(vTypes[nIndex],'\t').isEmpty())
+            {
+                /**Because of xrmex lexer there are duplicated id's,
+                   only use this if the lexer have already fixed*/
+                if (sActUnTrans.getToken(PoEntry::GROUPID,'\t')==
+                    sActUnTrans.getToken(PoEntry::LOCALID,'\t') &&
+                    sActUnTrans.getToken(PoEntry::SOURCEFILE,'\t').
+                                endsWith(".xrm"))
+                {
+                    sActUnTrans = DelLocalId(sActUnTrans);
+                }
+                PoEntry aPE(sActUnTrans, vTypes[nIndex]);
+                aPE.setTransStr(sActTrans.getToken(vTypes[nIndex],'\t'));
+                aPE.setFuzzy(sActTrans.isEmpty() ? 0 :
+                             bool(sActTrans.getToken(PoEntry::DUMMY,'\t').
+                                            copy(nDummyBit++,1).toInt32()));
+                aPE.writeToFile(aOutPut);
+            }
+        }
 
         //Check wheather next entry is in the same po file
         OString sNextSourcePath = GetPath(sPath,sLine);
@@ -145,12 +184,10 @@ int main(int argc, char* argv[])
     //Usage
     if (argc < 4)
     {
-        cout << "Use: renewpot translationsdir po2oo en-us.sdf" << endl;
-        cout << "translationsdir: in this dir there are language" << endl;
-        cout << "directories which contain the po files. These" << endl;
-        cout << "directories have named by the languageid" << endl;
-        cout << "po2oo: the path withwhich po2oo can be call" << endl;
-        cout << "en-us.sdf: the path to call po2oo with this sdf" << endl;
+        cout << "Use: renewpot translationsdir po2lo en-US.sdf" << endl;
+        cout << "translationsdir: this directory contains the po" << endl;
+        cout << "files of all languages. Every language has a" << endl;
+        cout << "directory named with language id." << endl;
         return 1;
     }