Package: release.debian.org Severity: normal User: release.debian....@packages.debian.org Usertags: unblock
Please unblock package gramps Bug #779308: Gramps 4.1.1 contains several bugs (originally reported upstream only) that render Gramps unusable for users of languages with special characters. This was due to the switch to Python 3 causing a database upgrade, and strings being converted in ASCII format instead of Unicode. The patch has been cherry-picked from upstream who have only recently fixed the problem. diff -Nru gramps-4.1.1~dfsg/debian/changelog gramps-4.1.1~dfsg/debian/changelog --- gramps-4.1.1~dfsg/debian/changelog 2014-12-08 19:33:21.000000000 +0100 +++ gramps-4.1.1~dfsg/debian/changelog 2015-02-26 21:59:03.000000000 +0100 @@ -1,3 +1,9 @@ +gramps (4.1.1~dfsg-3) unstable; urgency=medium + + * Add patch to fix bugs when database upgraded to Python 3 (Closes: #779308) + + -- Ross Gammon <rossgam...@mail.dk> Thu, 26 Feb 2015 21:57:31 +0100 + gramps (4.1.1~dfsg-2) unstable; urgency=medium * Fix clean target (Closes: #772573) diff -Nru gramps-4.1.1~dfsg/debian/patches/0002-PickleUpgradePython2to3.patch gramps-4.1.1~dfsg/debian/patches/0002-PickleUpgradePython2to3.patch --- gramps-4.1.1~dfsg/debian/patches/0002-PickleUpgradePython2to3.patch 1970-01-01 01:00:00.000000000 +0100 +++ gramps-4.1.1~dfsg/debian/patches/0002-PickleUpgradePython2to3.patch 2015-02-26 21:59:03.000000000 +0100 @@ -0,0 +1,251 @@ +From: Nick Hall <nic...@gramps-project.org> & Tim Lyons <guy.lin...@gmail.com> +Subject: Fix pickle upgrade of python2 strings to python3 str + +* Add gramps_upgrade_pickle() function: commit 27648255c0808ee3f097cd1e2debba5850079bbf +* Fix schema upgrade from 15 to 16 with python3: commit 03579aa4a1e386387e6daa23b4e2517b2e520109 +* Move load of metadata (which can also have pickle upgrade errors) and + make zip backup for pickle or schema upgrade: commit 2cd23356b4c91dae6fb8060c6f2e06236e03c45c +* Add missing import statement: commit 8cc86bd153e95ad1a9323ca894077b22d0232136 + +Origin: upstream, https://github.com/gramps- project/gramps/commits/maintenance/gramps41 +Bug: https://gramps-project.org/bugs/view.php?id=8134 +Bug-Debian: https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=779308 + +--- gramps.orig/gramps/gen/db/dbconst.py ++++ gramps/gramps/gen/db/dbconst.py +@@ -39,7 +39,7 @@ + ('DBPAGE', 'DBMODE', 'DBCACHE', 'DBLOCKS', 'DBOBJECTS', 'DBUNDO', + 'DBEXT', 'DBMODE_R', 'DBMODE_W', 'DBUNDOFN', 'DBLOCKFN', + 'DBRECOVFN','BDBVERSFN', 'DBLOGNAME', 'DBFLAGS_O', 'DBFLAGS_R', +- 'DBFLAGS_D', ++ 'DBFLAGS_D', 'SCHVERSFN', 'PCKVERSFN' + ) + + + ('PERSON_KEY', 'FAMILY_KEY', 'SOURCE_KEY', 'CITATION_KEY', +@@ -55,6 +55,8 @@ + DBLOCKFN = "lock" # File name of lock file + DBRECOVFN = "need_recover" # File name of recovery file + BDBVERSFN = "bdbversion.txt"# File name of Berkeley DB version file ++SCHVERSFN = "schemaversion.txt"# File name of schema version file ++PCKVERSFN = "pickleupgrade.txt" # Indicator that pickle has been upgrade t Python3 + DBLOGNAME = ".Db" # Name of logger + DBMODE_R = "r" # Read-only access + DBMODE_W = "w" # Full Read/Write access +--- gramps.orig/gramps/gen/db/read.py ++++ gramps/gramps/gen/db/read.py +@@ -1654,10 +1654,6 @@ + handle = handle.encode('utf-8') + try: + return table.get(handle, txn=self.txn) +- except UnicodeDecodeError: +- #we need to assume we opened data in python3 saved in python2 +- raw = table.db.get(handle, txn=self.txn) +- return pickle.loads(raw, encoding='utf-8') + except DBERRS as msg: + self.__log_error() + raise DbError(msg) +--- gramps.orig/gramps/gen/db/upgrade.py ++++ gramps/gramps/gen/db/upgrade.py +@@ -66,6 +66,27 @@ + + LOG = logging.getLogger(".upgrade") + ++def gramps_upgrade_pickle(self): ++ """ ++ Upgrade to python3 pickle protocol. ++ """ ++ import pickle ++ tables = (self.person_map, self.event_map, self.family_map, self.place_map, ++ self.repository_map, self.source_map, self.citation_map, ++ self.media_map, self.note_map, self.tag_map, self.metadata, ++ self.reference_map) ++ self.set_total(sum(map(len, tables))) ++ for data_map in tables: ++ for handle in data_map.keys(): ++ raw = data_map.db.get(handle) ++ data = pickle.loads(raw, encoding='utf-8') ++ with BSDDBTxn(self.env, data_map) as txn: ++ txn.put(handle, data) ++ self.update() ++ ++ with BSDDBTxn(self.env, self.metadata) as txn: ++ txn.put(b'upgraded', 'Yes') ++ + def gramps_upgrade_17(self): + """ + Upgrade database from version 16 to 17. +@@ -859,12 +880,12 @@ + new_citation = (new_handle, new_gramps_id, + date, page, confidence, ref, note_list, new_media_list, + new_data_map, new_change, private) ++ citation_list.append((new_handle)) + with BSDDBTxn(self.env, self.citation_map) as txn: + if isinstance(new_handle, UNITYPE): + new_handle = new_handle.encode('utf-8') + txn.put(new_handle, new_citation) + self.cmap_index += 1 +- citation_list.append((new_handle)) + return citation_list + + def gramps_upgrade_15(self): +--- gramps.orig/gramps/gen/db/write.py ++++ gramps/gramps/gen/db/write.py +@@ -396,6 +396,7 @@ + self.brief_name = None + self.update_env_version = False + self.update_python_version = False ++ self.update_pickle_version = False + + def catch_db_error(func): + """ +@@ -733,6 +734,25 @@ + force_bsddb_downgrade) + + self.__check_python_version(name, force_python_upgrade) ++ # Check for pickle upgrade ++ versionpath = os.path.join(self.path, cuni(PCKVERSFN)) ++ if sys.version_info[0] >= 3 and not os.path.isfile(versionpath) and \ ++ not self.readonly and not self.update_pickle_version: ++ _LOG.debug("Make backup in case there is a pickle upgrade") ++ self.__make_zip_backup(name) ++ self.update_pickle_version = True ++ ++ # Check for schema upgrade ++ versionpath = os.path.join(self.path, cuni(SCHVERSFN)) ++ if os.path.isfile(versionpath): ++ with open(versionpath, "r") as version_file: ++ schema_version = int(version_file.read().strip()) ++ else: ++ schema_version = 0 ++ if not self.readonly and schema_version < _DBVERSION and \ ++ force_schema_upgrade: ++ _LOG.debug("Make backup in case there is a schema upgrade") ++ self.__make_zip_backup(name) + + # Set up database environment + self.env = db.DBEnv() +@@ -788,7 +808,6 @@ + self.__close_early() + raise DbVersionError(tree_vers, _MINVERSION, _DBVERSION) + +- self.__load_metadata() + gstats = self.metadata.get(b'gender_stats', default=None) + + # Ensure version info in metadata +@@ -799,6 +818,7 @@ + # New database. Set up the current version. + #self.metadata.put(b'version', _DBVERSION, txn=the_txn) + txn.put(b'version', _DBVERSION) ++ txn.put(b'upgraded', 'Yes') + elif b'version' not in self.metadata: + # Not new database, but the version is missing. + # Use 0, but it is likely to fail anyway. +@@ -844,7 +864,7 @@ + if isinstance(version, UNITYPE): + version = version.encode('utf-8') + version_file.write(version) +- _LOG.debug("Updated BDBVERSFN file to %s" % str(db.version())) ++ _LOG.debug("Updated bsddb version file to %s" % str(db.version())) + + if self.update_python_version: + versionpath = os.path.join(name, "pythonversion.txt") +@@ -860,6 +880,21 @@ + # If secondary indices change, then they should removed + # or rebuilt by upgrade as well. In any case, the + # self.secondary_connected flag should be set accordingly. ++ if self.update_pickle_version: ++ from . import upgrade ++ UpdateCallback.__init__(self, callback) ++ upgrade.gramps_upgrade_pickle(self) ++ versionpath = os.path.join(name, cuni(PCKVERSFN)) ++ with open(versionpath, "w") as version_file: ++ version = "Yes" ++ if sys.version_info[0] <3: ++ if isinstance(version, UNITYPE): ++ version = version.encode('utf-8') ++ version_file.write(version) ++ _LOG.debug("Updated pickle version file to %s" % str(version)) ++ ++ self.__load_metadata() ++ + if self.need_schema_upgrade(): + oldschema = self.metadata.get(b'version', default=0) + newschema = _DBVERSION +@@ -867,6 +902,14 @@ + (oldschema, newschema)) + if force_schema_upgrade == True: + self.gramps_upgrade(callback) ++ versionpath = os.path.join(name, cuni(SCHVERSFN)) ++ with open(versionpath, "w") as version_file: ++ version = str(_DBVERSION) ++ if sys.version_info[0] <3: ++ if isinstance(version, UNITYPE): ++ version = version.encode('utf-8') ++ version_file.write(version) ++ _LOG.debug("Updated schema version file to %s" % str(version)) + else: + self.__close_early() + clear_lock_file(name) +@@ -936,13 +979,8 @@ + + # bookmarks + def meta(key): +- try: +- return self.metadata.get(key, default=[]) +- except UnicodeDecodeError: +- #we need to assume we opened data in python3 saved in python2 +- raw = self.metadata.db.get(key, default=[]) +- return pickle.loads(raw, encoding='utf-8') if raw else raw +- ++ return self.metadata.get(key, default=[]) ++ + self.bookmarks.set(meta(b'bookmarks')) + self.family_bookmarks.set(meta(b'family_bookmarks')) + self.event_bookmarks.set(meta(b'event_bookmarks')) +@@ -2135,10 +2173,6 @@ + handle = handle.encode('utf-8') + try: + data = data_map.get(handle, txn=self.txn) +- except UnicodeDecodeError: +- #we need to assume we opened data in python3 saved in python2 +- raw = data_map.db.get(handle, txn=self.txn) +- data = pickle.loads(raw, encoding='utf-8') + except: + data = None + # under certain circumstances during a database reload, +@@ -2330,8 +2364,9 @@ + version = self.metadata.get(b'version', default=_MINVERSION) + + t = time.time() +- ++ + from . import upgrade ++ + if version < 14: + upgrade.gramps_upgrade_14(self) + if version < 15: +@@ -2419,6 +2454,24 @@ + with open(versionpath, "w") as version_file: + version_file.write(version) + ++ versionpath = os.path.join(name, cuni(PCKVERSFN)) ++ _LOG.debug("Write pickle version file to %s" % "Yes") ++ with open(versionpath, "w") as version_file: ++ version = "Yes" ++ if sys.version_info[0] <3: ++ if isinstance(version, UNITYPE): ++ version = version.encode('utf-8') ++ version_file.write(version) ++ ++ versionpath = os.path.join(name, cuni(SCHVERSFN)) ++ _LOG.debug("Write schema version file to %s" % str(_DBVERSION)) ++ with open(versionpath, "w") as version_file: ++ version = str(_DBVERSION) ++ if sys.version_info[0] <3: ++ if isinstance(version, UNITYPE): ++ version = version.encode('utf-8') ++ version_file.write(version) ++ + versionpath = os.path.join(name, "pythonversion.txt") + version = str(version_info[0]) + if sys.version_info[0] < 3: diff -Nru gramps-4.1.1~dfsg/debian/patches/series gramps-4.1.1~dfsg/debian/patches/series --- gramps-4.1.1~dfsg/debian/patches/series 2014-12-08 19:33:21.000000000 +0100 +++ gramps-4.1.1~dfsg/debian/patches/series 2015-02-26 21:59:03.000000000 +0100 @@ -1 +1,2 @@ 0001-Correct-resource-path-in-setup.py.patch +0002-PickleUpgradePython2to3.patch unblock gramps/4.1.1~dfsg-3 -- System Information: Debian Release: jessie/sid APT prefers trusty-security APT policy: (990, 'trusty-security'), (900, 'trusty-updates'), (500, 'trusty'), (400, 'trusty-proposed'), (200, 'utopic-proposed'), (100, 'trusty- backports') Architecture: amd64 (x86_64) Foreign Architectures: i386 Kernel: Linux 3.13.0-46-generic (SMP w/4 CPU cores) -- To UNSUBSCRIBE, email to debian-bugs-dist-requ...@lists.debian.org with a subject of "unsubscribe". Trouble? Contact listmas...@lists.debian.org