From 2aadcf292dd3225740d0fd6fe051c6b5bc96c7ce Mon Sep 17 00:00:00 2001 From: Benoit Thiell Date: Fri, 3 Jun 2011 16:40:32 -0400 Subject: [PATCH] Merge multiple implementations of get_fieldvalues. Because of the problem caused by reciprocal imports, there are several implementations of get_fieldvalues in Invenio. I centralized a unique implementation in search_engine_utils. --- .../bibauthorid/lib/bibauthorid_tables_utils.py | 6 +- modules/bibauthorid/lib/bibauthorid_templates.py | 2 +- .../bibauthorid/lib/bibauthorid_webinterface.py | 3 +- .../bibcirculation/lib/bibcirculation_daemon.py | 2 +- modules/bibcirculation/lib/bibcirculation_utils.py | 2 +- modules/bibedit/lib/bibedit_utils.py | 6 +- modules/bibformat/lib/bibformat_dblayer.py | 15 +---- modules/bibformat/lib/bibformat_utils.py | 19 +----- modules/bibformat/lib/elements/bfe_issn.py | 5 +- modules/bibharvest/lib/oai_repository_updater.py | 6 +- modules/bibindex/lib/bibindex_engine.py | 12 +--- modules/bibmerge/lib/bibmerge_engine.py | 4 +- modules/bibrank/lib/bibrank_citation_indexer.py | 4 +- .../bibrank/lib/bibrank_downloads_similarity.py | 22 +------ modules/bibsword/lib/bibsword_webinterface.py | 2 +- modules/miscutil/demo/democfgdata.sql | 8 +- modules/webcomment/lib/webcomment.py | 2 +- modules/webcomment/lib/webcomment_templates.py | 2 +- modules/websearch/lib/Makefile.am | 5 +- modules/websearch/lib/search_engine.py | 50 +------------- modules/websearch/lib/search_engine_utils.py | 74 ++++++++++++++++++++ .../websearch/lib/websearch_regression_tests.py | 3 +- modules/websearch/lib/websearch_templates.py | 21 +----- modules/websearch/lib/websearch_webinterface.py | 2 +- modules/webstat/lib/webstat_engine.py | 2 +- .../doc/admin/websubmit-admin-guide.webdoc | 2 +- .../Mail_Approval_Request_to_Committee_Chair.py | 3 +- .../functions/User_is_Record_Owner_or_Curator.py | 2 +- modules/websubmit/web/publiline.py | 3 +- 29 files changed, 119 insertions(+), 170 deletions(-) create mode 100644 modules/websearch/lib/search_engine_utils.py diff --git a/modules/bibauthorid/lib/bibauthorid_tables_utils.py b/modules/bibauthorid/lib/bibauthorid_tables_utils.py index b183297..268825c 100644 --- a/modules/bibauthorid/lib/bibauthorid_tables_utils.py +++ b/modules/bibauthorid/lib/bibauthorid_tables_utils.py @@ -33,12 +33,12 @@ from bibauthorid_authorname_utils import update_doclist try: from search_engine import get_record - from search_engine import get_fieldvalues + from search_engine_utils import get_fieldvalues from bibrank_citation_searcher import get_citation_dict from dbquery import run_sql, run_sql_many from dbquery import OperationalError, ProgrammingError except ImportError: - from invenio.search_engine import get_fieldvalues + from invenio.search_engine_utils import get_fieldvalues from invenio.search_engine import get_record from invenio.bibrank_citation_searcher import get_citation_dict from invenio.dbquery import run_sql, run_sql_many @@ -1607,7 +1607,7 @@ def empty_aid_tables(): ''' Will empty all tables needed for a re-run of the algorithm. Exceptions are aidAUTHORNAMES*, which have to be updated apriori and - aidPERSONID, which has to be updated from algorithm after the re-run. + aidPERSONID, which has to be updated from algorithm after the re-run. ''' run_sql("TRUNCATE `aidDOCLIST`;" "TRUNCATE `aidREALAUTHORDATA`;" diff --git a/modules/bibauthorid/lib/bibauthorid_templates.py b/modules/bibauthorid/lib/bibauthorid_templates.py index 06782b2..40b2a6d 100644 --- a/modules/bibauthorid/lib/bibauthorid_templates.py +++ b/modules/bibauthorid/lib/bibauthorid_templates.py @@ -31,7 +31,7 @@ from invenio.config import CFG_SITE_URL from invenio.config import CFG_BIBAUTHORID_AUTHOR_TICKET_ADMIN_EMAIL from invenio.bibformat import format_record from invenio.session import get_session -from invenio.search_engine import get_fieldvalues +from invenio.search_engine_utils import get_fieldvalues from invenio.bibauthorid_webapi import get_bibref_name_string, get_person_redirect_link from invenio.messages import gettext_set_language, wash_language #from invenio.textutils import encode_for_xml diff --git a/modules/bibauthorid/lib/bibauthorid_webinterface.py b/modules/bibauthorid/lib/bibauthorid_webinterface.py index 70faa4e..8ef27b6 100644 --- a/modules/bibauthorid/lib/bibauthorid_webinterface.py +++ b/modules/bibauthorid/lib/bibauthorid_webinterface.py @@ -43,7 +43,8 @@ from invenio.webuser import email_valid_p, emailUnique from invenio.webuser import get_email_from_username, get_uid_from_email, isUserSuperAdmin from invenio.access_control_admin import acc_find_user_role_actions from invenio.access_control_admin import acc_get_user_roles, acc_get_role_id -from invenio.search_engine import perform_request_search, get_fieldvalues, sort_records +from invenio.search_engine import perform_request_search, sort_records +from invenio.search_engine_utils import get_fieldvalues import invenio.bibauthorid_webapi as webapi import invenio.bibauthorid_config as bconfig diff --git a/modules/bibcirculation/lib/bibcirculation_daemon.py b/modules/bibcirculation/lib/bibcirculation_daemon.py index b5dbc3c..f065c04 100644 --- a/modules/bibcirculation/lib/bibcirculation_daemon.py +++ b/modules/bibcirculation/lib/bibcirculation_daemon.py @@ -32,7 +32,7 @@ from invenio.mailutils import send_email import invenio.bibcirculation_dblayer as db from invenio.bibcirculation_config import CFG_BIBCIRCULATION_TEMPLATES, \ CFG_BIBCIRCULATION_LIBRARIAN_EMAIL -from invenio.search_engine import get_fieldvalues +from invenio.search_engine_utils import get_fieldvalues from invenio.bibcirculation_utils import generate_email_body diff --git a/modules/bibcirculation/lib/bibcirculation_utils.py b/modules/bibcirculation/lib/bibcirculation_utils.py index fdad98d..bd8ceb2 100644 --- a/modules/bibcirculation/lib/bibcirculation_utils.py +++ b/modules/bibcirculation/lib/bibcirculation_utils.py @@ -21,7 +21,7 @@ __revision__ = "$Id$" -from invenio.search_engine import get_fieldvalues +from invenio.search_engine_utils import get_fieldvalues from invenio.bibtask import task_low_level_submission import invenio.bibcirculation_dblayer as db from invenio.urlutils import create_html_link diff --git a/modules/bibedit/lib/bibedit_utils.py b/modules/bibedit/lib/bibedit_utils.py index fd64712..4c3476e 100644 --- a/modules/bibedit/lib/bibedit_utils.py +++ b/modules/bibedit/lib/bibedit_utils.py @@ -52,9 +52,9 @@ from invenio.dateutils import convert_datetext_to_dategui from invenio.bibedit_dblayer import get_bibupload_task_opts, \ get_marcxml_of_record_revision, get_record_revisions, \ get_info_of_record_revision -from invenio.search_engine import get_fieldvalues, print_record, \ - record_exists, get_colID, guess_primary_collection_of_a_record, \ - get_record +from invenio.search_engine import print_record, record_exists, get_colID, \ + guess_primary_collection_of_a_record, get_record +from invenio.search_engine_utils import get_fieldvalues from invenio.webuser import get_user_info from invenio.dbquery import run_sql from invenio.websearchadminlib import get_detailed_page_tabs diff --git a/modules/bibformat/lib/bibformat_dblayer.py b/modules/bibformat/lib/bibformat_dblayer.py index 0f45cd9..9f6b857 100644 --- a/modules/bibformat/lib/bibformat_dblayer.py +++ b/modules/bibformat/lib/bibformat_dblayer.py @@ -28,20 +28,7 @@ import zlib import time from invenio.dbquery import run_sql - -## MARC-21 tag/field access functions -def get_fieldvalues(recID, tag): - """Returns list of values of the MARC-21 'tag' fields for the record - 'recID'.""" - out = [] - bibXXx = "bib" + tag[0] + tag[1] + "x" - bibrec_bibXXx = "bibrec_" + bibXXx - query = "SELECT value FROM %s AS b, %s AS bb WHERE bb.id_bibrec=%s AND bb.id_bibxxx=b.id AND tag LIKE '%s'" \ - % (bibXXx, bibrec_bibXXx, recID, tag) - res = run_sql(query) - for row in res: - out.append(row[0]) - return out +from invenio.search_engine_utils import get_fieldvalues def localtime_to_utc(date, fmt="%Y-%m-%dT%H:%M:%SZ"): "Convert localtime to UTC" diff --git a/modules/bibformat/lib/bibformat_utils.py b/modules/bibformat/lib/bibformat_utils.py index 4f891c3..cdf5bc4 100644 --- a/modules/bibformat/lib/bibformat_utils.py +++ b/modules/bibformat/lib/bibformat_utils.py @@ -41,6 +41,7 @@ from invenio.dbquery import run_sql from invenio.urlutils import string_to_numeric_char_reference from invenio.textutils import encode_for_xml from invenio.shellutils import run_shell_command +from invenio.search_engine_utils import get_fieldvalues def highlight_matches(text, compiled_pattern, \ prefix_tag='', suffix_tag=""): @@ -172,24 +173,6 @@ def record_get_xml(recID, format='xm', decompress=zlib.decompress, """ from invenio.search_engine import record_exists - def get_fieldvalues(recID, tag): - """Return list of field values for field TAG inside record RECID.""" - out = [] - if tag == "001___": - # we have asked for recID that is not stored in bibXXx tables - out.append(str(recID)) - else: - # we are going to look inside bibXXx tables - digit = tag[0:2] - bx = "bib%sx" % digit - bibx = "bibrec_bib%sx" % digit - query = "SELECT bx.value FROM %s AS bx, %s AS bibx WHERE bibx.id_bibrec='%s' AND bx.id=bibx.id_bibxxx AND bx.tag LIKE '%s'" \ - "ORDER BY bibx.field_number, bx.tag ASC" % (bx, bibx, recID, tag) - res = run_sql(query) - for row in res: - out.append(row[0]) - return out - def get_creation_date(recID, fmt="%Y-%m-%d"): "Returns the creation date of the record 'recID'." out = "" diff --git a/modules/bibformat/lib/elements/bfe_issn.py b/modules/bibformat/lib/elements/bfe_issn.py index f78a6b2..86d2825 100644 --- a/modules/bibformat/lib/elements/bfe_issn.py +++ b/modules/bibformat/lib/elements/bfe_issn.py @@ -26,9 +26,8 @@ import urllib import sys import re import getopt -from invenio.search_engine import \ - get_fieldvalues, \ - perform_request_search +from invenio.search_engine import perform_request_search +from invenio.search_engine_utils import get_fieldvalues from invenio.config import CFG_CERN_SITE if CFG_CERN_SITE: diff --git a/modules/bibharvest/lib/oai_repository_updater.py b/modules/bibharvest/lib/oai_repository_updater.py index b4d0660..c3110d7 100644 --- a/modules/bibharvest/lib/oai_repository_updater.py +++ b/modules/bibharvest/lib/oai_repository_updater.py @@ -43,10 +43,8 @@ from invenio.config import \ CFG_BINDIR, \ CFG_SITE_NAME, \ CFG_TMPDIR -from invenio.search_engine import \ - perform_request_search, \ - get_fieldvalues, \ - get_record +from invenio.search_engine import perform_request_search, get_record +from invenio.search_engine_utils import get_fieldvalues from invenio.intbitset import intbitset as HitSet from invenio.dbquery import run_sql from invenio.bibtask import \ diff --git a/modules/bibindex/lib/bibindex_engine.py b/modules/bibindex/lib/bibindex_engine.py index 8551016..d5d427c 100644 --- a/modules/bibindex/lib/bibindex_engine.py +++ b/modules/bibindex/lib/bibindex_engine.py @@ -67,6 +67,7 @@ from invenio.intbitset import intbitset from invenio.errorlib import register_exception from invenio.htmlutils import remove_html_markup, get_links_in_html_page from invenio.textutils import wash_for_utf8 +from invenio.search_engine_utils import get_fieldvalues if sys.hexversion < 0x2040000: # pylint: disable=W0622 @@ -125,17 +126,6 @@ def kill_sleepy_mysql_threads(max_threads=CFG_MAX_MYSQL_THREADS, thread_timeout= write_message("WARNING: too many DB threads, killing thread %s" % r_id, verbose=1) return -## MARC-21 tag/field access functions -def get_fieldvalues(recID, tag): - """Returns list of values of the MARC-21 'tag' fields for the record - 'recID'.""" - bibXXx = "bib" + tag[0] + tag[1] + "x" - bibrec_bibXXx = "bibrec_" + bibXXx - query = "SELECT value FROM %s AS b, %s AS bb WHERE bb.id_bibrec=%%s AND bb.id_bibxxx=b.id AND tag LIKE %%s" \ - % (bibXXx, bibrec_bibXXx) - res = run_sql(query, (recID, tag)) - return [row[0] for row in res] - def get_associated_subfield_value(recID, tag, value, associated_subfield_code): """Return list of ASSOCIATED_SUBFIELD_CODE, if exists, for record RECID and TAG of value VALUE. Used by fulltext indexer only. diff --git a/modules/bibmerge/lib/bibmerge_engine.py b/modules/bibmerge/lib/bibmerge_engine.py index f3a2b81..e37029c 100644 --- a/modules/bibmerge/lib/bibmerge_engine.py +++ b/modules/bibmerge/lib/bibmerge_engine.py @@ -26,7 +26,8 @@ from invenio.bibmerge_merger import merge_field_group, replace_field, \ add_subfield, replace_subfield, \ delete_subfield, copy_R2_to_R1, merge_record from invenio.search_engine import print_record, perform_request_search, \ - get_fieldvalues + record_exists +from invenio.search_engine_utils import get_fieldvalues from invenio.bibedit_utils import cache_exists, cache_expired, \ create_cache_file, delete_cache_file, get_cache_file_contents, \ get_cache_mtime, latest_record_revision, record_locked_by_other_user, \ @@ -35,7 +36,6 @@ from invenio.bibedit_utils import cache_exists, cache_expired, \ get_record_revision_ids, revision_format_valid_p, split_revid, \ get_marcxml_of_revision_id from invenio.htmlutils import remove_html_markup -from invenio.search_engine import record_exists from invenio.bibrecord import create_record, record_xml_output, record_add_field from invenio.bibedit_config import CFG_BIBEDIT_TO_MERGE_SUFFIX diff --git a/modules/bibrank/lib/bibrank_citation_indexer.py b/modules/bibrank/lib/bibrank_citation_indexer.py index af94765..212c919 100644 --- a/modules/bibrank/lib/bibrank_citation_indexer.py +++ b/modules/bibrank/lib/bibrank_citation_indexer.py @@ -32,8 +32,8 @@ if sys.hexversion < 0x2040000: from invenio.dbquery import run_sql, serialize_via_marshal, \ deserialize_via_marshal -from invenio.search_engine import search_pattern, get_fieldvalues, \ - search_unit +from invenio.search_engine import search_pattern, search_unit +from invenio.search_engine_utils import get_fieldvalues from invenio.bibformat_utils import parse_tag from invenio.bibtask import write_message, task_get_option, \ task_update_progress, task_sleep_now_if_required, \ diff --git a/modules/bibrank/lib/bibrank_downloads_similarity.py b/modules/bibrank/lib/bibrank_downloads_similarity.py index 542c09f..8518aea 100644 --- a/modules/bibrank/lib/bibrank_downloads_similarity.py +++ b/modules/bibrank/lib/bibrank_downloads_similarity.py @@ -25,27 +25,7 @@ from invenio.config import \ CFG_CERN_SITE from invenio.dbquery import run_sql from invenio.bibrank_downloads_indexer import database_tuples_to_single_list - -def get_fieldvalues(recID, tag): - """Return list of field values for field TAG inside record RECID. - Copy from search_engine""" - out = [] - if tag == "001___": - # we have asked for recID that is not stored in bibXXx tables - out.append(str(recID)) - else: - # we are going to look inside bibXXx tables - digit = tag[0:2] - bx = "bib%sx" % digit - bibx = "bibrec_bib%sx" % digit - query = "SELECT bx.value FROM %s AS bx, %s AS bibx " \ - " WHERE bibx.id_bibrec='%s' AND bx.id=bibx.id_bibxxx AND " \ - " bx.tag LIKE '%s' ORDER BY bibx.field_number, bx.tag ASC" % \ - (bx, bibx, recID, tag) - res = run_sql(query) - for row in res: - out.append(row[0]) - return out +from invenio.search_engine_utils import get_fieldvalues def record_exists(recID): """Return 1 if record RECID exists. diff --git a/modules/bibsword/lib/bibsword_webinterface.py b/modules/bibsword/lib/bibsword_webinterface.py index a690bd8..c428632 100644 --- a/modules/bibsword/lib/bibsword_webinterface.py +++ b/modules/bibsword/lib/bibsword_webinterface.py @@ -39,7 +39,7 @@ from invenio.messages import gettext_set_language from invenio.webinterface_handler import wash_urlargd, WebInterfaceDirectory from invenio.websubmit_functions.Get_Recid import \ get_existing_records_for_reportnumber -from invenio.search_engine import get_fieldvalues +from invenio.search_engine_utils import get_fieldvalues from invenio.bibsword_config import CFG_MARC_REPORT_NUMBER, CFG_MARC_ADDITIONAL_REPORT_NUMBER class WebInterfaceSword(WebInterfaceDirectory): diff --git a/modules/miscutil/demo/democfgdata.sql b/modules/miscutil/demo/democfgdata.sql index 333f3d6..f3c2178 100644 --- a/modules/miscutil/demo/democfgdata.sql +++ b/modules/miscutil/demo/democfgdata.sql @@ -1192,15 +1192,15 @@ INSERT INTO sbmFIELDDESC VALUES ('DEMOJRN_AU',NULL,'100__a','T',NULL,4,60,NULL,N INSERT INTO sbmFIELDDESC VALUES ('DEMOJRN_EMAIL',NULL,'859__a','T',NULL,4,60,NULL,NULL,NULL,'2008-09-23','2009-02-20','

E-mail(s) of the author(s): (one per line)
',NULL,0); INSERT INTO sbmFIELDDESC VALUES ('DEMOJRN_TITLEE',NULL,'245__a','T',NULL,5,60,NULL,NULL,NULL,'2008-09-23','2009-02-20','

English title:
',NULL,0); INSERT INTO sbmFIELDDESC VALUES ('DEMOJRN_TITLEF',NULL,'246_1a','T',NULL,5,60,NULL,NULL,NULL,'2008-09-23','2009-02-20','

French title:
',NULL,0); -INSERT INTO sbmFIELDDESC VALUES ('DEMOJRN_ABSF',NULL,'590__b','R',NULL,100,90,NULL,NULL,'from invenio.htmlutils import get_html_text_editor\r\nfrom invenio.config import CFG_SITE_URL\r\nfrom invenio.search_engine import get_fieldvalues\r\nimport os\r\n\r\nif (\'modify\' in curdir) and not os.path.exists(\"%s/DEMOJRN_ABSF\" % curdir):\r\n try:\r\n content = get_fieldvalues(int(sysno), \'590__b\')[0]\r\n except:\r\n content = \'\'\r\nelif os.path.exists(\"%s/DEMOJRN_ABSE\" % curdir):\r\n content = file(\"%s/DEMOJRN_ABSE\" % curdir).read()\r\nelse:\r\n content = \'\'\r\n\r\ntext = get_html_text_editor(\"DEMOJRN_ABSF\", id=\"BulletinFCKEditor1\", content=content, toolbar_set=\"WebJournal\", width=\'522px\', height=\'700px\', file_upload_url=CFG_SITE_URL + \'/submit/attachfile\', custom_configurations_path=\'/fckeditor/journal-editor-config.js\')','2008-09-23','2009-02-23','
French Article:
',NULL,0); +INSERT INTO sbmFIELDDESC VALUES ('DEMOJRN_ABSF',NULL,'590__b','R',NULL,100,90,NULL,NULL,'from invenio.htmlutils import get_html_text_editor\r\nfrom invenio.config import CFG_SITE_URL\r\nfrom invenio.search_engine_utils import get_fieldvalues\r\nimport os\r\n\r\nif (\'modify\' in curdir) and not os.path.exists(\"%s/DEMOJRN_ABSF\" % curdir):\r\n try:\r\n content = get_fieldvalues(int(sysno), \'590__b\')[0]\r\n except:\r\n content = \'\'\r\nelif os.path.exists(\"%s/DEMOJRN_ABSE\" % curdir):\r\n content = file(\"%s/DEMOJRN_ABSE\" % curdir).read()\r\nelse:\r\n content = \'\'\r\n\r\ntext = get_html_text_editor(\"DEMOJRN_ABSF\", id=\"BulletinFCKEditor1\", content=content, toolbar_set=\"WebJournal\", width=\'522px\', height=\'700px\', file_upload_url=CFG_SITE_URL + \'/submit/attachfile\', custom_configurations_path=\'/fckeditor/journal-editor-config.js\')','2008-09-23','2009-02-23','
French Article:
',NULL,0); INSERT INTO sbmFIELDDESC VALUES ('DEMOJRN_CONT',NULL,'','D',NULL,NULL,NULL,NULL,NULL,'
\r\n\r\n
','2008-10-06','2008-10-06',NULL,NULL,0); INSERT INTO sbmFIELDDESC VALUES ('DEMOJRN_END',NULL,'','D',NULL,NULL,NULL,NULL,NULL,'
\r\n\r\n
','2008-09-23','2009-02-20','',NULL,0); -INSERT INTO sbmFIELDDESC VALUES ('DEMOJRN_ISSUES',NULL,'','R',NULL,NULL,NULL,NULL,NULL,'from invenio.search_engine import get_fieldvalues\r\nfrom invenio.webjournal_utils import get_next_journal_issues, get_current_issue, get_journal_issue_grouping\r\nimport os\r\n\r\norders_and_issues = [(\'\',\'\')]*4\r\n\r\nif (\'modify\' in curdir) and not os.path.exists(\"%s/DEMOJRN_ISSUE1\" % curdir):\r\n try:\r\n orders = get_fieldvalues(int(sysno), \'773__c\')\r\n issues = get_fieldvalues(int(sysno), \'773__n\')\r\n orders_and_issues = zip(orders, issues) + orders_and_issues\r\n except:\r\n pass\r\nelif (\'running\' in curdir) and not os.path.exists(\"%s/DEMOJRN_ISSUE1\" % curdir):\r\n try:\r\n journal_name = \'AtlantisTimes\'\r\n current_issue = get_current_issue(\'en\', journal_name)\r\n nb_issues = get_journal_issue_grouping(journal_name)\r\n next_issue_numbers = get_next_journal_issues(current_issue, journal_name, nb_issues)\r\n orders_and_issues = zip([\'\']*4, next_issue_numbers) + orders_and_issues\r\n except:\r\n pass\r\nissues_fields = []\r\nsingle_issue_and_order_tmpl = \'\'\'\r\n\r\n\'\'\'\r\ni = 1\r\nfor order_and_issue in orders_and_issues[:4]:\r\n order = order_and_issue[0]\r\n issue = order_and_issue[1]\r\n issues_fields.append(single_issue_and_order_tmpl % (i, order, i, issue))\r\n i += 1\r\n\r\ntext = \'
\'.join(issues_fields)\r\n','2009-02-20','2009-02-23','*Order(s) (digit) and issue(s) (xx/YYYY) of the article:
',NULL,0); +INSERT INTO sbmFIELDDESC VALUES ('DEMOJRN_ISSUES',NULL,'','R',NULL,NULL,NULL,NULL,NULL,'from invenio.search_engine_utils import get_fieldvalues\r\nfrom invenio.webjournal_utils import get_next_journal_issues, get_current_issue, get_journal_issue_grouping\r\nimport os\r\n\r\norders_and_issues = [(\'\',\'\')]*4\r\n\r\nif (\'modify\' in curdir) and not os.path.exists(\"%s/DEMOJRN_ISSUE1\" % curdir):\r\n try:\r\n orders = get_fieldvalues(int(sysno), \'773__c\')\r\n issues = get_fieldvalues(int(sysno), \'773__n\')\r\n orders_and_issues = zip(orders, issues) + orders_and_issues\r\n except:\r\n pass\r\nelif (\'running\' in curdir) and not os.path.exists(\"%s/DEMOJRN_ISSUE1\" % curdir):\r\n try:\r\n journal_name = \'AtlantisTimes\'\r\n current_issue = get_current_issue(\'en\', journal_name)\r\n nb_issues = get_journal_issue_grouping(journal_name)\r\n next_issue_numbers = get_next_journal_issues(current_issue, journal_name, nb_issues)\r\n orders_and_issues = zip([\'\']*4, next_issue_numbers) + orders_and_issues\r\n except:\r\n pass\r\nissues_fields = []\r\nsingle_issue_and_order_tmpl = \'\'\'\r\n\r\n\'\'\'\r\ni = 1\r\nfor order_and_issue in orders_and_issues[:4]:\r\n order = order_and_issue[0]\r\n issue = order_and_issue[1]\r\n issues_fields.append(single_issue_and_order_tmpl % (i, order, i, issue))\r\n i += 1\r\n\r\ntext = \'
\'.join(issues_fields)\r\n','2009-02-20','2009-02-23','*Order(s) (digit) and issue(s) (xx/YYYY) of the article:
',NULL,0); INSERT INTO sbmFIELDDESC VALUES ('DEMOJRN_RN',NULL,'037__a','I',35,NULL,NULL,NULL,'BUL---???',NULL,'2008-10-06','2009-02-20',NULL,NULL,0); INSERT INTO sbmFIELDDESC VALUES ('DEMOJRN_IN',NULL,'595__a','H',NULL,NULL,NULL,NULL,'Atlantis Times',NULL,'2008-09-23','2009-02-20',' ',NULL,0); -INSERT INTO sbmFIELDDESC VALUES ('DEMOJRN_ABSE',NULL,'520__b','R',NULL,100,90,NULL,NULL,'from invenio.htmlutils import get_html_text_editor\r\nfrom invenio.config import CFG_SITE_URL\r\nfrom invenio.search_engine import get_fieldvalues\r\nimport os\r\n\r\n\r\nif (\'modify\' in curdir) and not os.path.exists(\"%s/DEMOJRN_ABSE\" % curdir):\r\n try:\r\n content = get_fieldvalues(int(sysno), \'520__b\')[0]\r\n except:\r\n content = \'\'\r\nelif os.path.exists(\"%s/DEMOJRN_ABSE\" % curdir):\r\n content = file(\"%s/DEMOJRN_ABSE\" % curdir).read()\r\nelse:\r\n content = \'\'\r\n\r\ntext = get_html_text_editor(\"DEMOJRN_ABSE\",id=\"BulletinFCKEditor2\", content=content, toolbar_set=\"WebJournal\", width=\'522px\', height=\'700px\', file_upload_url=CFG_SITE_URL + \'/submit/attachfile\', custom_configurations_path=\'/fckeditor/journal-editor-config.js\')\r\n\r\n','2008-09-22','2009-02-23','
English Article:
',NULL,0); +INSERT INTO sbmFIELDDESC VALUES ('DEMOJRN_ABSE',NULL,'520__b','R',NULL,100,90,NULL,NULL,'from invenio.htmlutils import get_html_text_editor\r\nfrom invenio.config import CFG_SITE_URL\r\nfrom invenio.search_engine_utils import get_fieldvalues\r\nimport os\r\n\r\n\r\nif (\'modify\' in curdir) and not os.path.exists(\"%s/DEMOJRN_ABSE\" % curdir):\r\n try:\r\n content = get_fieldvalues(int(sysno), \'520__b\')[0]\r\n except:\r\n content = \'\'\r\nelif os.path.exists(\"%s/DEMOJRN_ABSE\" % curdir):\r\n content = file(\"%s/DEMOJRN_ABSE\" % curdir).read()\r\nelse:\r\n content = \'\'\r\n\r\ntext = get_html_text_editor(\"DEMOJRN_ABSE\",id=\"BulletinFCKEditor2\", content=content, toolbar_set=\"WebJournal\", width=\'522px\', height=\'700px\', file_upload_url=CFG_SITE_URL + \'/submit/attachfile\', custom_configurations_path=\'/fckeditor/journal-editor-config.js\')\r\n\r\n','2008-09-22','2009-02-23','
English Article:
',NULL,0); INSERT INTO sbmFIELDDESC VALUES ('DEMOJRN_ENDING',NULL,'','H',NULL,NULL,NULL,NULL,NULL,NULL,'2009-02-06','2009-02-20','',NULL,0); -INSERT INTO sbmFIELDDESC VALUES ('DEMOJRN_CATEG',NULL,'','R',NULL,NULL,NULL,NULL,NULL,'# Solve usual problem with submit/direct?.. links that bypass \r\n# the comboXXX (category selection) of the submission. Retrieve \r\n# the value, and set it (only in the case of MBI)\r\n\r\nfrom invenio.search_engine import get_fieldvalues\r\n\r\nif \"modify\" in curdir:\r\n try:\r\n comboDEMOJRNfile = file(\"%s/%s\" % (curdir,\'comboDEMOJRN\'), \'w\')\r\n report_number = get_fieldvalues(int(sysno), \'037__a\')[0]\r\n category = report_number.split(\'-\')[1]\r\n comboDEMOJRNfile.write(category)\r\n comboDEMOJRNfile.close()\r\n except:\r\n text = \'\'','2009-10-15','2009-10-15',NULL,NULL,0); +INSERT INTO sbmFIELDDESC VALUES ('DEMOJRN_CATEG',NULL,'','R',NULL,NULL,NULL,NULL,NULL,'# Solve usual problem with submit/direct?.. links that bypass \r\n# the comboXXX (category selection) of the submission. Retrieve \r\n# the value, and set it (only in the case of MBI)\r\n\r\nfrom invenio.search_engine_utils import get_fieldvalues\r\n\r\nif \"modify\" in curdir:\r\n try:\r\n comboDEMOJRNfile = file(\"%s/%s\" % (curdir,\'comboDEMOJRN\'), \'w\')\r\n report_number = get_fieldvalues(int(sysno), \'037__a\')[0]\r\n category = report_number.split(\'-\')[1]\r\n comboDEMOJRNfile.write(category)\r\n comboDEMOJRNfile.close()\r\n except:\r\n text = \'\'','2009-10-15','2009-10-15',NULL,NULL,0); INSERT INTO sbmFUNCTIONS VALUES ('SBI','DEMOPIC','Mail_Submitter',70,1); INSERT INTO sbmFUNCTIONS VALUES ('SBI','DEMOPIC','Print_Success',60,1); diff --git a/modules/webcomment/lib/webcomment.py b/modules/webcomment/lib/webcomment.py index c133c70..744dfd3 100644 --- a/modules/webcomment/lib/webcomment.py +++ b/modules/webcomment/lib/webcomment.py @@ -67,9 +67,9 @@ from invenio.access_control_engine import acc_authorize_action from invenio.search_engine import \ guess_primary_collection_of_a_record, \ check_user_can_view_record, \ - get_fieldvalues, \ get_collection_reclist, \ get_colID +from invenio.search_engine_utils import get_fieldvalues from invenio.webcomment_washer import EmailWasher try: import invenio.template diff --git a/modules/webcomment/lib/webcomment_templates.py b/modules/webcomment/lib/webcomment_templates.py index 79a2097..4d541f3 100644 --- a/modules/webcomment/lib/webcomment_templates.py +++ b/modules/webcomment/lib/webcomment_templates.py @@ -49,7 +49,7 @@ from invenio.htmlutils import get_html_text_editor from invenio.messages import gettext_set_language from invenio.bibformat import format_record from invenio.access_control_engine import acc_authorize_action -from invenio.websearch_templates import get_fieldvalues +from invenio.search_engine_utils import get_fieldvalues class Template: """templating class, refer to webcomment.py for examples of call""" diff --git a/modules/websearch/lib/Makefile.am b/modules/websearch/lib/Makefile.am index f74181b..78b3468 100644 --- a/modules/websearch/lib/Makefile.am +++ b/modules/websearch/lib/Makefile.am @@ -9,7 +9,7 @@ ## Invenio is distributed in the hope that it will be useful, but ## WITHOUT ANY WARRANTY; without even the implied warranty of ## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -## General Public License for more details. +## General Public License for more details. ## ## You should have received a copy of the GNU General Public License ## along with Invenio; if not, write to the Free Software Foundation, Inc., @@ -25,6 +25,7 @@ pylib_DATA = \ search_engine.py \ search_engine_config.py \ search_engine_tests.py \ + search_engine_utils.py \ search_engine_query_parser.py \ search_engine_query_parser_tests.py \ websearch_webcoll.py \ @@ -38,7 +39,7 @@ pylib_DATA = \ websearch_external_collections_searcher.py \ websearch_external_collections_templates.py \ websearch_external_collections_tests.py \ - websearch_external_collections_utils.py + websearch_external_collections_utils.py EXTRA_DIST = $(pylib_DATA) diff --git a/modules/websearch/lib/search_engine.py b/modules/websearch/lib/search_engine.py index 3d3370a..42f147b 100644 --- a/modules/websearch/lib/search_engine.py +++ b/modules/websearch/lib/search_engine.py @@ -74,6 +74,7 @@ from invenio.config import \ CFG_SITE_RECORD from invenio.search_engine_config import InvenioWebSearchUnknownCollectionError, InvenioWebSearchWildcardLimitError +from invenio.search_engine_utils import get_fieldvalues from invenio.bibrecord import create_record, record_get_field_instances from invenio.bibrank_record_sorter import get_bibrank_methods, rank_records, is_method_valid from invenio.bibrank_downloads_similarity import register_page_view_event, calculate_reading_similarity_list @@ -3110,55 +3111,6 @@ def get_field_tags(field): out.append(val[0]) return out - -def get_fieldvalues(recIDs, tag, repetitive_values=True): - """ - Return list of field values for field TAG for the given record ID - or list of record IDs. (RECIDS can be both an integer or a list - of integers.) - - If REPETITIVE_VALUES is set to True, then return all values even - if they are doubled. If set to False, then return unique values - only. - """ - out = [] - if isinstance(recIDs, (int, long)): - recIDs =[recIDs,] - if not isinstance(recIDs, (list, tuple)): - return [] - if len(recIDs) == 0: - return [] - if tag == "001___": - # we have asked for tag 001 (=recID) that is not stored in bibXXx tables - out = [str(recID) for recID in recIDs] - else: - # we are going to look inside bibXXx tables - digits = tag[0:2] - try: - intdigits = int(digits) - if intdigits < 0 or intdigits > 99: - raise ValueError - except ValueError: - # invalid tag value asked for - return [] - bx = "bib%sx" % digits - bibx = "bibrec_bib%sx" % digits - queryparam = [] - for recID in recIDs: - queryparam.append(recID) - if not repetitive_values: - queryselect = "DISTINCT(bx.value)" - else: - queryselect = "bx.value" - query = "SELECT %s FROM %s AS bx, %s AS bibx WHERE bibx.id_bibrec IN (%s) " \ - " AND bx.id=bibx.id_bibxxx AND bx.tag LIKE %%s " \ - " ORDER BY bibx.field_number, bx.tag ASC" % \ - (queryselect, bx, bibx, ("%s,"*len(queryparam))[:-1]) - res = run_sql(query, tuple(queryparam) + (tag,)) - for row in res: - out.append(row[0]) - return out - def get_fieldvalues_alephseq_like(recID, tags_in, can_see_hidden=False): """Return buffer of ALEPH sequential-like textual format with fields found in the list TAGS_IN for record RECID. diff --git a/modules/websearch/lib/search_engine_utils.py b/modules/websearch/lib/search_engine_utils.py new file mode 100644 index 0000000..177f470 --- /dev/null +++ b/modules/websearch/lib/search_engine_utils.py @@ -0,0 +1,74 @@ +# -*- coding: utf-8 -*- + +## This file is part of Invenio. +## Copyright (C) 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011 CERN. +## +## Invenio is free software; you can redistribute it and/or +## modify it under the terms of the GNU General Public License as +## published by the Free Software Foundation; either version 2 of the +## License, or (at your option) any later version. +## +## Invenio is distributed in the hope that it will be useful, but +## WITHOUT ANY WARRANTY; without even the implied warranty of +## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +## General Public License for more details. +## +## You should have received a copy of the GNU General Public License +## along with Invenio; if not, write to the Free Software Foundation, Inc., +## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA. + +# pylint: disable=C0301 + +"""Invenio search engine utilities.""" + +from invenio.dbquery import run_sql + +def get_fieldvalues(recIDs, tag, repetitive_values=True): + """ + Return list of field values for field TAG for the given record ID + or list of record IDs. (RECIDS can be both an integer or a list + of integers.) + + If REPETITIVE_VALUES is set to True, then return all values even + if they are doubled. If set to False, then return unique values + only. + """ + out = [] + if isinstance(recIDs, (int, long)): + recIDs = [recIDs,] + if not isinstance(recIDs, (list, tuple)): + return [] + if len(recIDs) == 0: + return [] + if tag == "001___": + # We have asked for tag 001 (=recID) that is not stored in bibXXx + # tables. + out = [str(recID) for recID in recIDs] + else: + # we are going to look inside bibXXx tables + digits = tag[0:2] + try: + intdigits = int(digits) + if intdigits < 0 or intdigits > 99: + raise ValueError + except ValueError: + # invalid tag value asked for + return [] + bx = "bib%sx" % digits + bibx = "bibrec_bib%sx" % digits + queryparam = [] + for recID in recIDs: + queryparam.append(recID) + if not repetitive_values: + queryselect = "DISTINCT(bx.value)" + else: + queryselect = "bx.value" + query = "SELECT %s FROM %s AS bx, %s AS bibx " \ + "WHERE bibx.id_bibrec IN (%s) AND bx.id=bibx.id_bibxxx AND " \ + "bx.tag LIKE %%s " \ + "ORDER BY bibx.field_number, bx.tag ASC" % \ + (queryselect, bx, bibx, ("%s,"*len(queryparam))[:-1]) + res = run_sql(query, tuple(queryparam) + (tag,)) + for row in res: + out.append(row[0]) + return out diff --git a/modules/websearch/lib/websearch_regression_tests.py b/modules/websearch/lib/websearch_regression_tests.py index 04803c9..b972050 100644 --- a/modules/websearch/lib/websearch_regression_tests.py +++ b/modules/websearch/lib/websearch_regression_tests.py @@ -46,8 +46,9 @@ from invenio.urlutils import same_urls_p from invenio.search_engine import perform_request_search, \ guess_primary_collection_of_a_record, guess_collection_of_a_record, \ collection_restricted_p, get_permitted_restricted_collections, \ - get_fieldvalues, search_pattern, search_unit, search_unit_in_bibrec, \ + search_pattern, search_unit, search_unit_in_bibrec, \ wash_colls +from invenio.search_engine_utils import get_fieldvalues def parse_url(url): parts = urlparse.urlparse(url) diff --git a/modules/websearch/lib/websearch_templates.py b/modules/websearch/lib/websearch_templates.py index 6685869..7dd57f5 100644 --- a/modules/websearch/lib/websearch_templates.py +++ b/modules/websearch/lib/websearch_templates.py @@ -76,30 +76,11 @@ from invenio.intbitset import intbitset from invenio.websearch_external_collections import external_collection_get_state, get_external_collection_engine from invenio.websearch_external_collections_utils import get_collection_id from invenio.websearch_external_collections_config import CFG_EXTERNAL_COLLECTION_MAXRESULTS +from invenio.search_engine_utils import get_fieldvalues _RE_PUNCTUATION = re.compile(CFG_BIBINDEX_CHARS_PUNCTUATION) _RE_SPACES = re.compile(r"\s+") -def get_fieldvalues(recID, tag): - """Return list of field values for field TAG inside record RECID. - FIXME: should be imported commonly for search_engine too.""" - out = [] - if tag == "001___": - # we have asked for recID that is not stored in bibXXx tables - out.append(str(recID)) - else: - # we are going to look inside bibXXx tables - digit = tag[0:2] - bx = "bib%sx" % digit - bibx = "bibrec_bib%sx" % digit - query = "SELECT bx.value FROM %s AS bx, %s AS bibx WHERE bibx.id_bibrec='%s' AND bx.id=bibx.id_bibxxx AND bx.tag LIKE '%s'" \ - "ORDER BY bibx.field_number, bx.tag ASC" % (bx, bibx, recID, tag) - res = run_sql(query) - for row in res: - out.append(row[0]) - return out - - class Template: # This dictionary maps Invenio language code to locale codes (ISO 639) diff --git a/modules/websearch/lib/websearch_webinterface.py b/modules/websearch/lib/websearch_webinterface.py index f34edba..8905657 100644 --- a/modules/websearch/lib/websearch_webinterface.py +++ b/modules/websearch/lib/websearch_webinterface.py @@ -102,7 +102,6 @@ from invenio.search_engine import check_user_can_view_record, \ create_similarly_named_authors_link_box, \ get_colID, \ get_coll_i18nname, \ - get_fieldvalues, \ get_fieldvalues_alephseq_like, \ get_most_popular_field_values, \ get_mysql_recid_from_aleph_sysno, \ @@ -113,6 +112,7 @@ from invenio.search_engine import check_user_can_view_record, \ perform_request_log, \ perform_request_search, \ restricted_collection_cache +from invenio.search_engine_utils import get_fieldvalues from invenio.access_control_engine import acc_authorize_action from invenio.access_control_config import VIEWRESTRCOLL from invenio.access_control_mailcookie import mail_cookie_create_authorize_action diff --git a/modules/webstat/lib/webstat_engine.py b/modules/webstat/lib/webstat_engine.py index bbfbb8e..aac94d4 100644 --- a/modules/webstat/lib/webstat_engine.py +++ b/modules/webstat/lib/webstat_engine.py @@ -28,8 +28,8 @@ from invenio.config import CFG_TMPDIR, CFG_SITE_URL, CFG_SITE_NAME, CFG_BINDIR from invenio.urlutils import redirect_to_url from invenio.search_engine import perform_request_search, \ get_collection_reclist, \ - get_fieldvalues, \ get_most_popular_field_values +from invenio.search_engine_utils import get_fieldvalues from invenio.dbquery import run_sql, \ wash_table_column_name from invenio.websubmitadmin_dblayer import get_docid_docname_alldoctypes diff --git a/modules/websubmit/doc/admin/websubmit-admin-guide.webdoc b/modules/websubmit/doc/admin/websubmit-admin-guide.webdoc index 860e3e7..1ec3e0e 100644 --- a/modules/websubmit/doc/admin/websubmit-admin-guide.webdoc +++ b/modules/websubmit/doc/admin/websubmit-admin-guide.webdoc @@ -785,7 +785,7 @@ A possible skeleton for a response element could be: (FIXME: Check...) 
 import os
 from invenio.websubmit_functions.ParamFile import ParamFromFile
-from invenio.search_engine import get_fieldvalues
+from invenio.search_engine_utils import get_fieldvalues
 
 this_element_name = "DEMOART_TEST" # This would be your element name
 
diff --git a/modules/websubmit/lib/functions/Mail_Approval_Request_to_Committee_Chair.py b/modules/websubmit/lib/functions/Mail_Approval_Request_to_Committee_Chair.py
index 38bb3a7..b31801c 100644
--- a/modules/websubmit/lib/functions/Mail_Approval_Request_to_Committee_Chair.py
+++ b/modules/websubmit/lib/functions/Mail_Approval_Request_to_Committee_Chair.py
@@ -44,7 +44,8 @@ from invenio.config import CFG_SITE_NAME, \
 
 from invenio.mailutils import send_email
 from invenio.access_control_admin import acc_get_role_id, acc_get_role_users
-from invenio.search_engine import search_pattern, get_fieldvalues
+from invenio.search_engine import search_pattern
+from invenio.search_engine_utils import get_fieldvalues
 from invenio.dbquery import run_sql
 
 #Copied from publiline
diff --git a/modules/websubmit/lib/functions/User_is_Record_Owner_or_Curator.py b/modules/websubmit/lib/functions/User_is_Record_Owner_or_Curator.py
index d7737c4..96f987d 100644
--- a/modules/websubmit/lib/functions/User_is_Record_Owner_or_Curator.py
+++ b/modules/websubmit/lib/functions/User_is_Record_Owner_or_Curator.py
@@ -25,7 +25,7 @@ __revision__ = "$Id$"
 import os
 
 from invenio.config import CFG_CERN_SITE
-from invenio.search_engine import get_fieldvalues
+from invenio.search_engine_utils import get_fieldvalues
 from invenio.websubmit_config import InvenioWebSubmitFunctionStop
 from invenio.access_control_engine import acc_authorize_action
 from invenio.access_control_admin import acc_get_role_id, acc_is_user_in_role
diff --git a/modules/websubmit/web/publiline.py b/modules/websubmit/web/publiline.py
index bbbfc3f..1225898 100644
--- a/modules/websubmit/web/publiline.py
+++ b/modules/websubmit/web/publiline.py
@@ -51,7 +51,8 @@ from invenio.webpage import page, create_error_box
 from invenio.webuser import getUid, get_email, page_not_authorized, collect_user_info
 from invenio.messages import gettext_set_language, wash_language
 #from invenio.websubmit_config import *
-from invenio.search_engine import search_pattern, get_fieldvalues,check_user_can_view_record
+from invenio.search_engine import search_pattern, check_user_can_view_record
+from invenio.search_engine_utils import get_fieldvalues
 from invenio.websubmit_functions.Retrieve_Data import Get_Field
 from invenio.mailutils import send_email
 from invenio.urlutils import wash_url_argument
-- 
1.7.4.1