commit: 932910d804e09718fe1d7e6e7a50d4d943557a66 Author: Magnus Granberg <zorry <AT> gentoo <DOT> org> AuthorDate: Mon Jul 20 15:45:59 2015 +0000 Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org> CommitDate: Mon Jul 20 15:45:59 2015 +0000 URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=932910d8
add CategoriesMetadata tbc/pym/db_mapping.py | 7 +++++++ tbc/pym/sqlquerys.py | 19 ++++++++++++++++--- tbc/pym/updatedb.py | 15 ++++++++++++++- 3 files changed, 37 insertions(+), 4 deletions(-) diff --git a/tbc/pym/db_mapping.py b/tbc/pym/db_mapping.py index 05fccae..cf4a054 100644 --- a/tbc/pym/db_mapping.py +++ b/tbc/pym/db_mapping.py @@ -67,6 +67,13 @@ class Categories(Base): TimeStamp = Column('time_stamp', DateTime, nullable=False, default=datetime.datetime.utcnow) __tablename__ = 'categories' +class CategoriesMetadata(Base): + Id = Column('id', Integer, primary_key=True) + CategoryId = Column('category_id', Integer, ForeignKey('categories.category_id')) + Checksum = Column('checksum', String(100)) + Descriptions = Column('descriptions', Text) + __tablename__ = 'categories_metadata' + class Repos(Base): RepoId = Column('repo_id', Integer, primary_key=True) Repo = Column('repo', String(100)) diff --git a/tbc/pym/sqlquerys.py b/tbc/pym/sqlquerys.py index 9d7189d..a521dbc 100644 --- a/tbc/pym/sqlquerys.py +++ b/tbc/pym/sqlquerys.py @@ -8,7 +8,7 @@ from tbc.db_mapping import Configs, Logs, ConfigsMetaData, Jobs, BuildJobs, Pack Uses, ConfigsEmergeOptions, EmergeOptions, HiLight, BuildLogs, BuildLogsConfig, BuildJobsUse, BuildJobsRedo, \ HiLightCss, BuildLogsHiLight, BuildLogsEmergeOptions, BuildLogsErrors, ErrorsInfo, EmergeInfo, BuildLogsUse, \ BuildJobsEmergeOptions, EbuildsMetadata, EbuildsIUse, Restrictions, EbuildsRestrictions, EbuildsKeywords, \ - Keywords, PackagesMetadata, Emails, PackagesEmails, Setups, BuildLogsRepomanQa + Keywords, PackagesMetadata, Emails, PackagesEmails, Setups, BuildLogsRepomanQa, CategoriesMetadata from sqlalchemy.orm.exc import NoResultFound, MultipleResultsFound from sqlalchemy import and_, or_ @@ -323,10 +323,23 @@ def update_repo_db(session, repo_list): session.add(Repos(Repo = repo)) session.commit() -def update_categories_db(session, category): - if not get_category_info(session, category): +def update_categories_db(session, category, categories_metadataDict): + CategoryInfo = get_category_info(session, category) + if not CategoryInfo: session.add(Categories(Category = category)) session.commit() + CategoryInfo = get_category_info(session, category) + try: + CategoriesMetadataInfo = session.query(CategoriesMetadata).filter_by(CategoryId = CategoryInfo.CategoryId).one() + except NoResultFound as e: + NewCategoriesMetadata = CategoriesMetadata(CategoryId = CategoryInfo.CategoryId, Checksum = categories_metadataDict['metadata_xml_checksum'], Descriptions = categories_metadataDict['metadata_xml_descriptions']) + session.add(NewCategoriesMetadata) + session.commit() + return + if CategoriesMetadataInfo.Checksum != categories_metadataDict['metadata_xml_checksum']: + CategoriesMetadataInfo.Checksum = categories_metadataDict['metadata_xml_checksum'] + CategoriesMetadataInfo.Descriptions = categories_metadataDict['metadata_xml_descriptions'] + session.commit() def get_keyword_id(session, keyword): try: diff --git a/tbc/pym/updatedb.py b/tbc/pym/updatedb.py index 1cacc7d..b8be53f 100644 --- a/tbc/pym/updatedb.py +++ b/tbc/pym/updatedb.py @@ -7,6 +7,7 @@ import os import multiprocessing import time import portage +from portage.xml.metadata import MetaDataXML from sqlalchemy.orm import scoped_session, sessionmaker from tbc.ConnectionManager import NewConnection from tbc.sqlquerys import add_logs, get_package_info, update_repo_db, \ @@ -36,6 +37,15 @@ def init_portage_settings(session, config_id): add_logs(session, log_msg, "info", config_id) return mysettings +def get_categories_metadataDict(pkgdir): + # Make categories_metadataDict + categories_metadataDict = {} + pkg_md = MetaDataXML(pkgdir + "/metadata.xml", None) + metadata_xml_descriptions_tree = re.sub('\t', '', pkg_md.descriptions()[0]) + categories_metadataDict['metadata_xml_descriptions'] = re.sub('\n', '', metadata_xml_descriptions_tree) + categories_metadataDict['metadata_xml_checksum'] = portage.checksum.sha256hash(pkgdir + "/metadata.xml")[0] + return categories_metadataDict + def update_cpv_db_pool(mysettings, myportdb, cp, repo, tbc_settings, config_id): session_factory = sessionmaker(bind=NewConnection(tbc_settings)) Session = scoped_session(session_factory) @@ -48,7 +58,10 @@ def update_cpv_db_pool(mysettings, myportdb, cp, repo, tbc_settings, config_id): package = element[1] # update the categories table - update_categories_db(session2, categories) + repodir = self._myportdb.getRepositoryPath('gentoo') + pkgdir = repodir + "/" + categories + categories_metadataDict = get_categories_metadataDict(pkgdir) + update_categories_db(session2, categories, categories_metadataDict) # Check if we have the cp in the package table PackagesInfo = get_package_info(session2, categories, package, repo)