Python polib.pofile() Examples

def import_lang(self, lang):
        po = polib.pofile(LOCALE_PATH.format(lang=lang))
        total = 0
        for entry in [e for e in po]:
            msgid = entry.msgid
            params = {"text_%s" % lang: entry.msgstr}
            total += (
                self.dbsession.query(HazCat)
                .filter(HazCat.general_recommendation == msgid)
                .update({"general_recommendation_%s" % lang: entry.msgstr})
            )
            total += self.dbsession.query(TecRec).filter(TecRec.text == msgid).update(params)
            total += (
                self.dbsession.query(TecRec)
                .filter(TecRec.detail == msgid)
                .update({"detail_%s" % lang: entry.msgstr})
            )
            total += self.dbsession.query(CcRec).filter(CcRec.text == msgid).update(params)
        print("[%s] %s strings updated" % (lang, total))
        self.dbsession.flush() 

def testPOT(potfile):
    """Run all tests against all entries in a POT file.

       :param str potfile: The name of a .pot file to test
       :return: whether the checks succeeded or not
       :rtype: bool
    """
    success = True

    parsed_pot = polib.pofile(potfile)

    for entry in parsed_pot:
        if not testString(entry):
            success = False

    return success 

def assert_merge_headers(self, file_path, num_headers):
        """
        This is invoked by test_main to ensure that it runs after
        calling generate.main().

        There should be exactly num_headers merge comment headers
        in our merged .po file. This counts them to be sure.
        A merge comment looks like this:
        # #-#-#-#-#  django-partial.po (0.1a)  #-#-#-#-#

        """
        pof = pofile(file_path)
        pattern = re.compile('^#-#-#-#-#', re.M)
        match = pattern.findall(pof.header)
        self.assertEqual(
            len(match),
            num_headers,
            msg="Found %s (should be %s) merge comments in the header for %s" % (len(match), num_headers, file_path)
        ) 

def build_js_translations(self):
        import polib
        import simplejson as json

        def create_js(source, lang):
            print 'Generating', lang
            po = polib.pofile(source)
            data = self.po2dict(po, lang)
            data = json.dumps(data, sort_keys=True,
                              ensure_ascii=False, indent=2 * ' ')
            out_dir = os.path.abspath(os.path.join(self.ckan_path, 'public',
                                                   'base', 'i18n'))
            out_file = open(os.path.join(out_dir, '%s.js' % lang), 'w')
            out_file.write(data.encode('utf-8'))
            out_file.close()

        for l in os.listdir(self.i18n_path):
            if os.path.isdir(os.path.join(self.i18n_path, l)):
                f = os.path.join(self.i18n_path, l, 'LC_MESSAGES', 'ckan.po')
                create_js(f, l)
        print 'Completed generating JavaScript translations' 

def make_dummy(filename, locale, converter):
    """
    Takes a source po file, reads it, and writes out a new po file
    in :param locale: containing a dummy translation.
    """
    if not Path(filename).exists():
        raise IOError(u'File does not exist: %r' % filename)
    pofile = polib.pofile(filename)
    for msg in pofile:
        # Some strings are actually formatting strings, don't dummy-ify them,
        # or dates will look like "DÀTÉ_TÌMÉ_FÖRMÀT Ⱡ'σ# EST"
        if is_format_message(msg):
            continue
        converter.convert_msg(msg)

    pofile.metadata['Language'] = locale

    # Apply declaration for English pluralization rules so that ngettext will
    # do something reasonable.
    pofile.metadata['Plural-Forms'] = 'nplurals=2; plural=(n != 1);'

    new_file = new_filename(filename, locale)
    new_file.parent.makedirs_p()
    pofile.save(new_file)
    clean_pofile(new_file) 

def check_po_file(path):
    errors = []

    def check_translation(validator, msgid, msgstr):
        if not validator(msgid) == validator(msgstr):
            errors.append((msgid, msgstr))

    po = polib.pofile(path)
    for entry in po.translated_entries():
        if entry.msgid_plural and entry.msgstr_plural:
            for function in (simple_conv_specs, mapping_keys,
                             replacement_fields):
                for key, msgstr in entry.msgstr_plural.iteritems():
                    if key == '0':
                        check_translation(function, entry.msgid,
                                          entry.msgstr_plural[key])
                    else:
                        check_translation(function, entry.msgid_plural,
                                          entry.msgstr_plural[key])
        elif entry.msgstr:
            for function in (simple_conv_specs, mapping_keys,
                             replacement_fields):
                check_translation(function, entry.msgid, entry.msgstr)

    return errors 

def translate_file(self, root, file_name, target_language):
        """
        convenience method for translating a pot file

        :param root:            the absolute path of folder where the file is present
        :param file_name:       name of the file to be translated (it should be a pot file)
        :param target_language: language in which the file needs to be translated
        """
        logger.info('filling up translations for locale `{}`'.format(target_language))

        po = polib.pofile(os.path.join(root, file_name))
        strings = self.get_strings_to_translate(po)

        # translate the strings,
        # all the translated strings are returned
        # in the same order on the same index
        # viz. [a, b] -> [trans_a, trans_b]
        tl = get_translator()
        translated_strings = tl.translate_strings(strings, target_language, 'en', False)
        self.update_translations(po, translated_strings)
        po.save() 

def get_new_header(configuration, pofile):
    """
    Insert info about edX into the po file headers
    """
    team = pofile.metadata.get('Language-Team', None)
    if not team:
        return TRANSIFEX_HEADER.format(configuration.TRANSIFEX_URL)
    return TRANSIFEX_HEADER.format(team) 

def assert_pofile_same(self, pofile1, pofile2):
        """The paths `p1` and `p2` should be identical pofiles."""
        po1 = polib.pofile(pofile1)
        po2 = polib.pofile(pofile2)
        self.assertEqual(po1, po2) 

def test_sample_data(self):
        work_file = WORK / "django.po"
        shutil.copyfile(TEST_DATA / "django_before.po", work_file)
        original_pofile = polib.pofile(work_file)

        written = segment_pofile(
            work_file,
            {
                'studio.po': [
                    'cms/*',
                    'other_cms/*',
                ],
            }
        )

        self.assertEqual(written, set([WORK / "django.po", WORK / "studio.po"]))

        pofiles = [polib.pofile(f) for f in written]
        after_entries = sum(len(pofile) for pofile in pofiles)
        self.assertEqual(len(original_pofile), after_entries)

        original_ids = set(m.msgid for m in original_pofile)
        after_ids = set(m.msgid for pofile in pofiles for m in pofile)
        self.assertEqual(original_ids, after_ids)

        self.assert_pofile_same(WORK / "django.po", TEST_DATA / "django_after.po")
        self.assert_pofile_same(WORK / "studio.po", TEST_DATA / "studio.po") 

def test_headers(self):
        """Verify all headers have been modified"""
        for path in self.get_files():
            po = polib.pofile(path)
            header = po.header
            self.assertTrue(
                'openedx-translation@googlegroups.com' in header,
                msg='Missing header in %s:\n"%s"' % (path, header)
            ) 

def test_metadata(self):
        """Verify all metadata has been modified"""
        for path in self.get_files():
            po = polib.pofile(path)
            metadata = po.metadata
            value = metadata['Report-Msgid-Bugs-To']
            expected = 'openedx-translation@googlegroups.com'
            self.assertEquals(expected, value) 

def clean_file(configuration, filename):
    """
    Strips out the warning from a translated po file about being an English source file.
    Replaces warning with a note about coming from Transifex.
    """
    pofile = polib.pofile(filename)

    if pofile.header.find(EDX_MARKER) != -1:
        new_header = get_new_header(configuration, pofile)
        new = pofile.header.replace(EDX_MARKER, new_header)
        pofile.header = new
        pofile.save() 

def test_msgfmt(pofile):
    # Check that the .po file can actually be compiled
    with tempfile.NamedTemporaryFile(mode="w+b", suffix=".mo") as mofile:
        try:
            # Ignore the output on success
            subprocess.check_output(["msgfmt", "-c", "--verbose", "-o", mofile.name, pofile],
                                    stderr=subprocess.STDOUT, universal_newlines=True)
        except subprocess.CalledProcessError as e:
            raise AssertionError("Unable to compile %s: %s" % (pofile, e.output)) 

def fix_header(pofile):
    """
    Replace default headers with edX headers
    """

    # By default, django-admin.py makemessages creates this header:
    #
    #   SOME DESCRIPTIVE TITLE.
    #   Copyright (C) YEAR THE PACKAGE'S COPYRIGHT HOLDER
    #   This file is distributed under the same license as the PACKAGE package.
    #   FIRST AUTHOR <EMAIL@ADDRESS>, YEAR.

    pofile.metadata_is_fuzzy = []   # remove [u'fuzzy']
    header = pofile.header
    fixes = (
        ('SOME DESCRIPTIVE TITLE', EDX_MARKER),
        ('Translations template for PROJECT.', EDX_MARKER),
        ('YEAR', str(datetime.utcnow().year)),
        ('ORGANIZATION', 'edX'),
        ("THE PACKAGE'S COPYRIGHT HOLDER", "EdX"),
        (
            'This file is distributed under the same license as the PROJECT project.',
            'This file is distributed under the GNU AFFERO GENERAL PUBLIC LICENSE.'
        ),
        (
            'This file is distributed under the same license as the PACKAGE package.',
            'This file is distributed under the GNU AFFERO GENERAL PUBLIC LICENSE.'
        ),
        ('FIRST AUTHOR <EMAIL@ADDRESS>', 'EdX Team <info@edx.org>'),
    )
    for src, dest in fixes:
        header = header.replace(src, dest)
    pofile.header = header 

def fix_metadata(pofile):
    """
    Replace default metadata with edX metadata
    """

    # By default, django-admin.py makemessages creates this metadata:
    #
    #   {u'PO-Revision-Date': u'YEAR-MO-DA HO:MI+ZONE',
    #   u'Language': u'',
    #   u'Content-Transfer-Encoding': u'8bit',
    #   u'Project-Id-Version': u'PACKAGE VERSION',
    #   u'Report-Msgid-Bugs-To': u'',
    #   u'Last-Translator': u'FULL NAME <EMAIL@ADDRESS>',
    #   u'Language-Team': u'LANGUAGE <LL@li.org>',
    #   u'POT-Creation-Date': u'2013-04-25 14:14-0400',
    #   u'Content-Type': u'text/plain; charset=UTF-8',
    #   u'MIME-Version': u'1.0'}

    fixes = {
        'PO-Revision-Date': datetime.utcnow(),
        'Report-Msgid-Bugs-To': 'openedx-translation@googlegroups.com',
        'Project-Id-Version': '0.1a',
        'Language': 'en',
        'Last-Translator': '',
        'Language-Team': 'openedx-translation <openedx-translation@googlegroups.com>',
        'Plural-Forms': 'nplurals=2; plural=(n != 1);',
    }
    pofile.metadata.update(fixes) 

def __init__(self, source):

        def get_pot_path(source_name):
            # when fileobj is a TemporaryFile, its name is an inter in P3, a string in P2
            if isinstance(source_name, str) and source_name.endswith('.po'):
                # Normally the path looks like /path/to/xxx/i18n/lang.po
                # and we try to find the corresponding
                # /path/to/xxx/i18n/xxx.pot file.
                # (Sometimes we have 'i18n_extra' instead of just 'i18n')
                path = Path(source_name)
                filename = path.parent.parent.name + '.pot'
                pot_path = path.with_name(filename)
                return pot_path.exists() and str(pot_path) or False
            return False

        # polib accepts a path or the file content as a string, not a fileobj
        if isinstance(source, str):
            self.pofile = polib.pofile(source)
            pot_path = get_pot_path(source)
        else:
            # either a BufferedIOBase or result from NamedTemporaryFile
            self.pofile = polib.pofile(source.read().decode())
            pot_path = get_pot_path(source.name)

        if pot_path:
            # Make a reader for the POT file
            # (Because the POT comments are correct on GitHub but the
            # PO comments tends to be outdated. See LP bug 933496.)
            self.pofile.merge(polib.pofile(pot_path)) 

def generate_mo(extraction_target_path, lang_name, trans_mappings, domain, localedir=None):
    """
    Extracts strings from 'extraction_target_path', and creates pot, po, mo file with
    'trans_mappings' information.
    'extraction_target_path' can be file or directory.
    """
    extraction_target_dir = extraction_target_path\
        if os.path.isdir(extraction_target_path) else os.path.dirname(extraction_target_path)
    localedir = localedir if localedir is not None \
                          else os.path.join(extraction_target_dir, 'locale')
    mo_dir = os.path.join(localedir, lang_name, 'LC_MESSAGES')
    create_dir([extraction_target_dir, 'locale', lang_name, 'LC_MESSAGES'])

    pot_file = '{0}.pot'.format(os.path.join(localedir, domain))
    po_file = '{0}.po'.format(os.path.join(mo_dir, domain))
    mo_file = '{0}.mo'.format(os.path.join(mo_dir, domain))

    extract_command = "pybabel extract {0} -o {1}".format(extraction_target_path, pot_file)
    utility.exec_command(extract_command, extraction_target_dir)

    po = polib.pofile(pot_file)
    for entry in po:
        if entry.msgid in trans_mappings:
            entry.msgstr = trans_mappings[entry.msgid]
    po.save(po_file)
    po.save_as_mofile(mo_file)
    return domain, localedir 

def backup_po_to_db(self):
        """ Backup Po file to db model """

        for lang, lang_name in settings.LANGUAGES:
            for path in settings.LOCALE_PATHS:
                po_pattern = os.path.join(path, get_dirname_from_lang(lang), "LC_MESSAGES", "*.po")
                for pofile in glob(po_pattern):
                    logger.debug("Backuping file {} to db".format(pofile))

                    domain = os.path.splitext(os.path.basename(pofile))[0]
                    with codecs.open(pofile, 'r', 'utf-8') as pofile_opened:
                        content = pofile_opened.read()
                        backup = TranslationBackup(
                            language=lang,
                            locale_path=get_relative_locale_path(pofile),
                            domain=domain,
                            locale_parent_dir=get_locale_parent_dirname(pofile),
                            content=content,
                        )
                        backup.save()

                    if get_settings('TRANSLATIONS_CLEAN_PO_AFTER_BACKUP'):
                        with open(pofile, 'w') as pofile_opened:
                            pofile_opened.write('')

    ############################################################################ 

def load_data_from_po(self):
        import os

        for lang, lang_name in settings.LANGUAGES:
            for path in settings.LOCALE_PATHS:
                locale = get_dirname_from_lang(lang)
                po_pattern = os.path.join(path, locale, "LC_MESSAGES", "*.po")
                for pofile in glob(po_pattern):
                    logger.debug("Processing pofile {}".format(pofile))
                    self.store_to_db(pofile=pofile, locale=locale, store_translations=True)

        self.postprocess() 

def test_usability(pofile):
    # Use polib to write a mofile
    with tempfile.NamedTemporaryFile(mode="w+b") as mofile:
        pofile = polib.pofile(pofile)
        pofile.save_as_mofile(mofile.name)

        # Try to open it
        _t = gettext.GNUTranslations(fp=mofile) 

def restore(self):
        po_filename = os.path.join(self.locale_path, self.language, 'LC_MESSAGES',
                                   self.domain + '.mo')

        mo_filename = os.path.join(self.locale_path, self.language, 'LC_MESSAGES',
                                   self.domain + '.mo')

        with open(po_filename, 'wb') as output:
            output.write(self.content.encode('utf-8'))

        po = polib.pofile(po_filename)
        po.save_as_mofile(mo_filename) 

def setUp(self):
        cmd = Command()
        cmd.set_options(**dict(
                locale='ia',
                set_fuzzy=False,
                skip_translated=False
        ))
        self.cmd = cmd
        self.po = polib.pofile(os.path.join(os.path.dirname(__file__), 'data/django.po')) 

def parseTranslation(langCode: str, poFile: str) -> dict:
  if langCode not in LOCALES:
    raise Exception("Language not installed")
  po = polib.pofile(poFile)
  result = {}
  for entry in po:
    result[entry.msgid] = entry.msgstr
  return result

# Build the translation one time to prevent them from being generated on each request 

def print_usage():
    """Print usage message and exit."""
    print 'Usage: po_to_mo <pofile> (or drag pofile onto executable icon).'
    raw_input()
    sys.exit(1) 

def get_translation_percentage(self, locale_path: str, locale: str) -> int:

        # backend stats
        po = polib.pofile(self.get_po_filename(locale_path, locale))
        not_translated = len(po.untranslated_entries())
        total = len(po.translated_entries()) + not_translated

        # frontend stats
        with open(self.get_json_filename(locale_path, locale)) as reader:
            for key, value in ujson.load(reader).items():
                total += 1
                if value == '':
                    not_translated += 1

        # mobile stats
        with open(os.path.join(locale_path, 'mobile_info.json')) as mob:
            mobile_info = ujson.load(mob)
        try:
            info = mobile_info[locale]
        except KeyError:
            if self.strict:
                raise
            info = {'total': 0, 'not_translated': 0}

        total += info['total']
        not_translated += info['not_translated']

        return (total - not_translated) * 100 // total 

def test_percentage(pofile):
    pofile = polib.pofile(pofile)
    if pofile.percent_translated() < threshold:
        # Issue a warning instead of an exception, since these should probably
        # be handled on a case-by-case basis
        warnings.warn("amount translated of %d%% below threshold of %d%%" % (pofile.percent_translated(), threshold)) 

def update_po_files():
    def get_current_msgids():
        pot = polib.pofile("locale/en/LC_MESSAGES/django.po")
        return {e.msgid for e in pot if not e.fuzzy and not e.obsolete}

    # get the current set of msgids
    saved_msgids = get_current_msgids()

    # re-extract locale files from source code
    ignore_paths = ("env/*", "fabric/*", "media/*", "sitestatic/*", "static/*", "node_modules/*")
    ignore_args = " ".join([f'--ignore="{p}"' for p in ignore_paths])

    cmd(f"python manage.py makemessages -a -e haml,html,txt,py --no-location --no-wrap {ignore_args}")

    # get the new set of msgids
    actual_msgids = get_current_msgids()

    added_msgids = actual_msgids.difference(saved_msgids)
    removed_msgids = saved_msgids.difference(actual_msgids)

    if DEBUG:
        for mid in added_msgids:
            print(f"  + {repr(mid)}")
        for mid in removed_msgids:
            print(f"  - {repr(mid)}")

    # if there are no actual changes to msgids, revert
    if not added_msgids and not removed_msgids:
        cmd("git checkout -- locale") 

def test_markup(pofile):
    po = polib.pofile(pofile)

    for entry in po.translated_entries():
        if is_markup(entry.msgid):
            # If this is a plural, check each of the plural translations
            if entry.msgid_plural:
                xlations = entry.msgstr_plural
            else:
                xlations = {None: entry.msgstr}

            for plural_id, msgstr in xlations.items():
                # Check if the markup is valid at all
                try:
                    # pylint: disable=unescaped-markup
                    ET.fromstring('<markup>%s</markup>' % msgstr)
                except ET.ParseError:
                    if entry.msgid_plural:
                        raise AssertionError("Invalid markup translation for %d translation of msgid %s\n%s" %
                                (plural_id, entry.msgid, msgstr))
                    else:
                        raise AssertionError("Invalid markup translation for msgid %s\n%s" %
                                (entry.msgid, msgstr))

                # Check if the markup has the same number and kind of tags
                if not markup_match(entry.msgid, msgstr):
                    if entry.msgid_plural:
                        raise AssertionError("Markup does not match for %d translation of msgid %s\n%s" %
                                (plural_id, entry.msgid, msgstr))
                    else:
                        raise AssertionError("Markup does not match for msgid %s\n%s" % (entry.msgid, msgstr)) 

def __init__(self, resxen2name, resxlocale, potemplate, out_path, code):
        if os.path.isfile(resxen2name) is False:
            raise Exception("Resx bound not found")

        if os.path.isfile(resxlocale) is False:
            raise Exception("Resx trans not found")

        if os.path.isfile(potemplate) is False:
            raise Exception("PO template not found")

        bound = resx2dict(resxen2name)
        trans = resx2dict(resxlocale)

        motable = {}
        for boundk in sorted(bound, key=len, reverse=True):
            if boundk in trans:
                motable[bound[boundk]] = trans[boundk]
            else:
                print("WARNING: {} not found in {}, wont be added into motable".format(boundk, resxen2name))

        po = polib.pofile(potemplate)

        po.metadata = {
            'Project-Id-Version': '1.0',
            'Report-Msgid-Bugs-To': 'adam.schubert@sg1-game.net',
            'POT-Creation-Date': '2016-01-19 02:00+0100',
            'PO-Revision-Date': '2016-01-19 02:00+0100',
            'Last-Translator': 'Adam Schubert <adam.schubert@sg1-game.net>',
            'Language-Team': '',
            'MIME-Version': '1.0',
            'Content-Type': 'text/plain; charset=utf-8',
            'Content-Transfer-Encoding': '8bit',
            'Language': code
        }
        for entry in po:
            if entry.msgid in motable:
                entry.msgstr = motable[entry.msgid]
            else:
                print("WARNING: {} not found in {}".format(entry.msgid, 'motable'))

        po.save()
        po.save_as_mofile(os.path.join(out_path, '{}.mo'.format(code)))
        shutil.copy2(potemplate, os.path.join(out_path, '{}.po'.format(code)))