BibConvert: modify oaiharvest xsl template

Makefile.am

@@ -205,6 +205,10 @@ install-jquery-plugins:
 	wget -N --no-check-certificate http://invenio-software.org/download/jquery/parsley.js &&\
 	wget -N --no-check-certificate http://invenio-software.org/download/jquery/spin.min.js &&\
 	rm -f jquery.bookmark.package-1.4.0.zip && \
+	wget https://cdnjs.cloudflare.com/ajax/libs/handlebars.js/1.3.0/handlebars.min.js && \
+	wget https://twitter.github.com/typeahead.js/releases/0.10.5/typeahead.bundle.min.js && \
+	wget https://raw.githubusercontent.com/es-shims/es5-shim/v4.0.3/es5-shim.min.js && \
+	wget https://raw.githubusercontent.com/es-shims/es5-shim/v4.0.3/es5-shim.map && \
 	mkdir -p ${prefix}/var/www/img && \
 	cd ${prefix}/var/www/img && \
 	wget -r -np -nH --cut-dirs=4 -A "png,css" -P jquery-ui/themes http://jquery-ui.googlecode.com/svn/tags/1.8.17/themes/base/ && \
@@ -236,6 +240,10 @@ uninstall-jquery-plugins:
 	rm -f jquery.dataTables.min.js && \
 	rm -f ui.core.js && \
 	rm -f jquery.bookmark.min.js && \
+	rm -f handlebars.min.js && \
+	rm -f typeahead.bundle.min.js && \
+	rm -f es5-shim.min.js && \
+	rm -f es5-shim.map && \
 	rm -f jquery.dataTables.ColVis.min.js && \
 	rm -f jquery.hotkeys.js && \
 	rm -f jquery.tablesorter.min.js && \
@@ -324,6 +332,60 @@ uninstall-pdfa-helper-files:
 	@echo "** The PDF/A helper files were successfully uninstalled. **"
 	@echo "***********************************************************"
 
+install-youtube:
+	@echo "***********************************************************"
+	@echo "**        Installing youtube client libraries            **"
+	@echo "***********************************************************"
+	@echo "Please make sure that you have pip installed             **"
+	@echo "-----------------------------------------------------------"
+	@echo "For more infos about the library please visit:"
+	@echo "https://developers.google.com/api-client-library/python/start/installation"
+	sudo pip install --upgrade google-api-python-client
+	rm -rf /tmp/invenio_js_frameworks
+	mkdir -p /tmp/invenio_js_frameworks
+	(cd /tmp/invenio_js_frameworks && \
+	wget https://github.com/dimsemenov/Magnific-Popup/archive/master.zip && \
+	unzip master.zip && \
+	mkdir -p ${prefix}/var/www/static/magnific_popup && \
+	cp -r Magnific-Popup-master/dist/* ${prefix}/var/www/static/magnific_popup && \
+	cd /tmp && \
+	rm -rf invenio_js_frameworks)
+	@echo "***********************************************************"
+	@echo "**  Youtube client libraries was successfully installed  **"
+	@echo "***********************************************************"
+
+unistall-youtube:
+	@echo "***********************************************************"
+	@echo "**       Unistalling Youtube client libraries            **"
+	@echo "***********************************************************"
+	sudo pip uninstall google-api-python-client
+	rm -rf ${prefix}/var/www/static/magnific_popup
+	@echo "***********************************************************"
+	@echo "** Youtube client libraries was successfully unistalled  **"
+
+install-webcomment:
+	@echo "***********************************************************"
+	@echo "** Installing Webcomment plugin dependencies.            **"
+	@echo "***********************************************************"
+	rm -rf /tmp/webcomment
+	mkdir /tmp/webcomment
+	wget 'https://github.com/cowboy/jquery-throttle-debounce/archive/master.zip' -O '/tmp/webcomment/webcomment.zip' --no-check-certificate
+	unzip -u -d '/tmp/webcomment' '/tmp/webcomment/webcomment.zip'
+	mv /tmp/webcomment/jquery-throttle-debounce-master/jquery.ba-throttle-debounce.min.js ${prefix}/var/www/js
+	wget 'https://raw.githubusercontent.com/bartaz/sandbox.js/master/jquery.highlight.js' -O '/tmp/webcomment/jquery.highlight.min.js' --no-check-certificate
+	mv /tmp/webcomment/jquery.highlight.min.js ${prefix}/var/www/js
+	rm -rf /tmp/webcomment
+	@echo "***********************************************************"
+	@echo "** Webcomment plugins were successfully installed.       **"
+	@echo "***********************************************************"
+
+uninstall-webcomment:
+	rm -f ${prefix}/var/www/js/jquery.ba-throttle-debounce.min.js
+	rm -f ${prefix}/var/www/js/jquery.highlight.min.js
+	@echo "***********************************************************"
+	@echo "** The Webcomment plugins were successfully uninstalled. **"
+	@echo "***********************************************************"
+
 #Solrutils allows automatic installation, running and searching of an external Solr index.
 install-solrutils:
 	@echo "***********************************************************"

config/invenio.conf

@@ -1171,6 +1171,8 @@ CFG_BIBINDEX_PERFORM_OCR_ON_DOCNAMES = scan-.*
 # NOTE: for backward compatibility reasons you can set this to a simple
 # regular expression that will directly be used as the unique key of the
 # map, with corresponding value set to ".*" (in order to match any URL)
+# NOTE2: If the value is None, the url mapping the key regex will be used
+# directly
 CFG_BIBINDEX_SPLASH_PAGES = {
     "http://documents\.cern\.ch/setlink\?.*": ".*",
     "http://ilcagenda\.linearcollider\.org/subContributionDisplay\.py\?.*|http://ilcagenda\.linearcollider\.org/contributionDisplay\.py\?.*": "http://ilcagenda\.linearcollider\.org/getFile\.py/access\?.*|http://ilcagenda\.linearcollider\.org/materialDisplay\.py\?.*",
@@ -1475,6 +1477,15 @@ CFG_WEBCOMMENT_MAX_ATTACHED_FILES = 5
 # discussions.
 CFG_WEBCOMMENT_MAX_COMMENT_THREAD_DEPTH = 1
 
+# CFG_WEBCOMMENT_ENABLE_HTML_EMAILS -- if True, emails will also contain
+# HTML content, in addition to the plaintext version.
+CFG_WEBCOMMENT_ENABLE_HTML_EMAILS = True
+
+# CFG_WEBCOMMENT_ENABLE_MARKDOWN_TEXT_RENDERING -- if True, and when
+# CFG_WEBCOMMENT_USE_RICH_TEXT_EDITOR is False, plain text will be rendered
+# as Markdown <http://daringfireball.net/projects/markdown/>.
+CFG_WEBCOMMENT_ENABLE_MARKDOWN_TEXT_RENDERING = True
+
 ##################################
 # Part 11: BibSched parameters ##
 ##################################
@@ -2565,6 +2576,50 @@ CFG_ARXIV_URL_PATTERN = http://export.arxiv.org/pdf/%sv%s.pdf
 # e.g. CFG_REDIS_HOSTS = [{'db': 0, 'host': '127.0.0.1', 'port': 7001}]
 CFG_REDIS_HOSTS = {'default': [{'db': 0, 'host': '127.0.0.1', 'port': 6379}]}
 
+#################################
+## Elasticsearch Configuration ##
+#################################
+
+## CFG_ELASTICSEARCH_LOGGING -- Whether to use Elasticsearch logging or not
+CFG_ELASTICSEARCH_LOGGING = 0
+
+## CFG_ELASTICSEARCH_INDEX_PREFIX -- The prefix to be used for the
+## Elasticsearch indices.
+CFG_ELASTICSEARCH_INDEX_PREFIX = invenio-
+
+## CFG_ELASTICSEARCH_HOSTS -- The list of Elasticsearch hosts to connect to.
+## This is a list of dictionaries with connection information.
+CFG_ELASTICSEARCH_HOSTS = [{'host': '127.0.0.1', 'port': 9200}]
+
+## CFG_ELASTICSEARCH_SUFFIX_FORMAT -- The time format string to base the
+## suffixes for the Elasticsearch indices on.  E.g. "%Y.%m" for indices to be
+## called "invenio-2014.10" for example.
+CFG_ELASTICSEARCH_SUFFIX_FORMAT = %Y.%m
+
+## CFG_ELASTICSEARCH_MAX_QUEUE_LENGTH -- The maximum length the queue of events
+## is allowed to grow to before it is flushed to Elasticsearch.  If you don't
+## want to set a maximum, and rely entirely on the periodic flush instead, set
+## this to -1.
+CFG_ELASTICSEARCH_MAX_QUEUE_LENGTH = -1
+
+## CFG_ELASTICSEARCH_FLUSH_INTERVAL -- The time (in seconds) to wait between
+## flushes of the event queue to Elasticsearch.  If you want to disable
+## periodic flushing and instead rely on the max. queue length to trigger
+## flushes, set this to -1.
+CFG_ELASTICSEARCH_FLUSH_INTERVAL = 30
+
+## CFG_ELASTICSEARCH_BOT_AGENT_STRINGS -- A list of strings which, if found in
+## the user agent string, will cause a 'bot' flag to be added to the logged
+## event.  This list taken from bots marked "active" at
+## <http://www.searchenginedictionary.com/spider-names.shtml>.  Googlebot and
+## bingbot added to the head of the list for speed.
+CFG_ELASTICSEARCH_BOT_AGENT_STRINGS = ['Googlebot', 'bingbot', 'Arachnoidea',
+'FAST-WebCrawler', 'Fluffy the spider', 'Gigabot', 'Gulper', 'ia_archiver',
+'MantraAgent', 'MSN', 'Scooter', 'Scrubby', 'Slurp', 'Teoma_agent1', 'Winona',
+'ZyBorg', 'Almaden', 'Cyveillance', 'DTSearch', 'Girafa.com', 'Indy Library',
+'LinkWalker', 'MarkWatch', 'NameProtect', 'Robozilla', 'Teradex Mapper',
+'Tracerlock', 'W3C_Validator', 'WDG_Validator', 'Zealbot']
+
 ##########################
 #  THAT's ALL, FOLKS!  ##
 ##########################

configure.ac

@@ -811,6 +811,7 @@ AC_CONFIG_FILES([config.nice  \
      modules/miscutil/etc/ckeditor_scientificchar/lang/Makefile \
      modules/miscutil/lib/Makefile \
      modules/miscutil/lib/upgrades/Makefile \
+     modules/miscutil/lib/pid_providers/Makefile \
      modules/miscutil/sql/Makefile \
      modules/miscutil/web/Makefile \
      modules/webaccess/Makefile \
@@ -874,6 +875,10 @@ AC_CONFIG_FILES([config.nice  \
      modules/webmessage/doc/hacking/Makefile \
      modules/webmessage/lib/Makefile \
      modules/webmessage/web/Makefile \
+     modules/webnews/Makefile \
+     modules/webnews/doc/Makefile \
+     modules/webnews/lib/Makefile \
+     modules/webnews/web/Makefile \
      modules/websearch/Makefile \
      modules/websearch/bin/Makefile \
      modules/websearch/bin/webcoll \
@@ -923,6 +928,7 @@ AC_CONFIG_FILES([config.nice  \
      modules/websubmit/etc/Makefile \
      modules/websubmit/lib/Makefile \
      modules/websubmit/lib/functions/Makefile \
+     modules/websubmit/lib/author_sources/Makefile \
      modules/websubmit/web/Makefile \
      modules/websubmit/web/admin/Makefile \
      modules/docextract/Makefile \

modules/Makefile.am

@@ -52,6 +52,7 @@ SUBDIRS = bibauthorid \
           webjournal \
           weblinkback \
           webmessage \
+          webnews \
           websearch \
           websession \
           webstat \

modules/bibcirculation/lib/bibcirculation_config.py

@@ -149,6 +149,7 @@
                     'We will process your order of the document immediately and will contact you as soon as it is delivered.\n\n'\
                     'Best regards,\nCERN Library team\n',
 
+
     'PURCHASE_RECEIVED_TID': 'Dear colleague,\n\n'\
                     'The document you requested has been received. '\
                     'The price is %s'\
@@ -221,7 +222,6 @@
                'Thank you in advance for your cooperation, CERN Library Staff',
     'EMPTY': 'Please choose one template'
     }
-
 else:
     CFG_BIBCIRCULATION_TEMPLATES = {
         'OVERDUE': 'Overdue letter template (write some text)',
@@ -375,8 +375,20 @@
         'EMPTY': 'Please choose one template'
         }
 
+ill_conf = ('Dear colleague,\n\n'
+            'We have received your interlibrary loan request\n'
+            '\tTitle: {0}\n\n'
+            'We will process your order of the document immediately and will '
+            'contact you as soon as it is delivered.\n\n'
+            'If you have any questions about your request, please contact '
+            '{1}\n\n'
+            'Best regards,\n'
+            'CERN Library team')
+
+CFG_BIBCIRCULATION_TEMPLATES['ILL_CONFIRMATION'] = ill_conf
+
 if CFG_CERN_SITE == 1:
-    CFG_BIBCIRCULATION_ILLS_EMAIL = 'CERN External loans<external.loans@cern.ch>'
+    CFG_BIBCIRCULATION_ILLS_EMAIL = 'CERN External loans<lib.acq@cern.ch>'
     CFG_BIBCIRCULATION_LIBRARIAN_EMAIL = 'CERN Library Desk<library.desk@cern.ch>'
     CFG_BIBCIRCULATION_LOANS_EMAIL = 'CERN Lib loans<lib.loans@cern.ch>'
 else:

modules/bibcirculation/lib/bibcirculation_daemon.py

@@ -23,23 +23,30 @@
 
 __revision__ = "$Id$"
 
+import os
 import sys
 import time
+import tempfile
+from invenio.config import CFG_TMPDIR
 from invenio.dbquery import run_sql
 from invenio.bibtask import task_init, \
                             task_sleep_now_if_required, \
+                            task_low_level_submission, \
                             task_update_progress, \
                             task_set_option, \
                             task_get_option, \
                             write_message
 from invenio.mailutils import send_email
+from invenio.search_engine_utils import get_fieldvalues
 import invenio.bibcirculation_dblayer as db
 from invenio.bibcirculation_config import CFG_BIBCIRCULATION_TEMPLATES, \
                                           CFG_BIBCIRCULATION_LOANS_EMAIL, \
                                           CFG_BIBCIRCULATION_ILLS_EMAIL, \
                                           CFG_BIBCIRCULATION_REQUEST_STATUS_WAITING, \
                                           CFG_BIBCIRCULATION_LOAN_STATUS_EXPIRED
 
+from invenio.config import CFG_BIBCIRCULATION_ITEM_STATUS_ON_SHELF, \
+                           CFG_BIBCIRCULATION_ITEM_STATUS_ON_LOAN
 from invenio.bibcirculation_utils import generate_email_body, \
                                          book_title_from_MARC, \
                                          update_user_info_from_ldap, \
@@ -59,6 +66,8 @@ def task_submit_elaborate_specific_parameter(key, value, opts, args):
         task_set_option('update-borrowers', True)
     elif key in ('-r', '--update-requests'):
         task_set_option('update-requests', True)
+    elif key in ('-p', '--add-physical-copies-shelf-number-to-marc'):
+        task_set_option('add-physical-copies-shelf-number-to-marc', True)
     else:
         return False
     return True
@@ -252,17 +261,80 @@ def task_run_core():
         task_update_progress("ILL recall: processed %d out of %d expired ills." % (done+1, total_expired_ills))
         write_message("Processed %d out of %d expired ills." % (done+1, total_expired_ills))
 
+    if task_get_option("add-physical-copies-shelf-number-to-marc"):
+        write_message("Started adding info. reg. physical copies and shelf number to records")
+        modified_rec_locs = db.get_modified_items_physical_locations()
+        #Tagging of records
+        if modified_rec_locs:
+            total_modified_rec_locs = len(modified_rec_locs)
+            MARC_RECS_STR = "<?xml version='1.0' encoding='UTF-8'?>\n<collection>"
+            recids_seen = []
+            for done, (recid, status, location, collection) in enumerate(modified_rec_locs):
+                if not int(recid) or not location or status not in [ CFG_BIBCIRCULATION_ITEM_STATUS_ON_SHELF, \
+                   CFG_BIBCIRCULATION_ITEM_STATUS_ON_LOAN ]  or collection=='periodical' or\
+                   recid in recids_seen or 'DELETED' in get_fieldvalues(recid, '980__c'):
+                   #or location in get_fieldvalues(recid, '852__h'):
+                    continue
+                #MARC_RECS_STR: Compose a string with the records containing the controlfield(recid) and
+                #the 2 datafields(shelf no, physical copies) for each item retrieved from the query
+                copies = db.get_item_copies_details(recid)
+                MARC_RECS_STR += '<record><controlfield tag="001">' + str(recid) + '</controlfield>'
+                type_copies = get_fieldvalues(recid, '340__a')
+                if 'paper' not in type_copies:
+                    MARC_RECS_STR += '<datafield tag="340" ind1=" " ind2=" "> \
+                                      <subfield code="a">paper</subfield> \
+                                      </datafield>'
+                    if 'ebook' in type_copies or 'e-book' in type_copies:
+                        MARC_RECS_STR += '<datafield tag="340" ind1=" " ind2=" "> \
+                                          <subfield code="a">ebook</subfield> \
+                                          </datafield>'
+                lib_loc_tuples = []
+                for (_barcode, _loan_period, library_name, _library_id,
+                     location, _nb_requests,  _status, _collection,
+                     _description, _due_date) in copies:
+                    if not library_name or not location: continue
+                    if not (library_name, location) in lib_loc_tuples:
+                        lib_loc_tuples.append((library_name, location))
+                    else: continue
+                    MARC_RECS_STR += '<datafield tag="852" ind1=" " ind2=" "> \
+                                      <subfield code="c">' + library_name + '</subfield> \
+                                      <subfield code="h">' + location.replace('&', ' and ') +'</subfield> \
+                                      </datafield>'
+                MARC_RECS_STR += '</record>'
+                recids_seen.append(recid)
+                # Upload chunks of 100 records and sleep if needed
+                if (done+1)%100 == 0 or (done+1) == total_modified_rec_locs:
+                    MARC_RECS_STR += "</collection>"
+                    timestamp = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.localtime())
+                    marcxmlfile = 'MARCxml_booksearch' + '_' + timestamp + '_'
+                    fd, marcxmlfile = tempfile.mkstemp(dir=CFG_TMPDIR, prefix=marcxmlfile, suffix='.xml')
+                    os.write(fd, MARC_RECS_STR)
+                    os.close(fd)
+                    write_message("Composed MARCXML saved into %s" % marcxmlfile)
+                    #Schedule the bibupload task.
+                    task_id = task_low_level_submission("bibupload", "BibCirc", "-c", marcxmlfile, '-P', '-3')
+                    write_message("BibUpload scheduled with task id %s" % task_id)
+                    write_message("Processed %d out of %d modified record locations." % (done+1, total_modified_rec_locs))
+                    MARC_RECS_STR = "<?xml version='1.0' encoding='UTF-8'?>\n<collection>"
+                    task_sleep_now_if_required(can_stop_too=True)
+
+        else:
+            write_message("No new records modified. Not scheduling any bibupload task")
+
     return 1
 
+
 def main():
 
     task_init(authorization_action='runbibcircd',
               authorization_msg="BibCirculation Task Submission",
               help_specific_usage="""-o,  --overdue-letters\tCheck overdue loans and send recall emails if necessary.\n
 -b,  --update-borrowers\tUpdate borrowers information from ldap.\n
--r,  --update-requests\tUpdate pending requests of users\n\n""",
+-r,  --update-requests\tUpdate pending requests of users\n
+-p,  --add-physical-copies-shelf-number-to-marc\tAdd info. reg. physical copies and shelf number to records' marc\n\n""",
               description="""Example: %s -u admin \n\n""" % (sys.argv[0]),
-              specific_params=("obr", ["overdue-letters", "update-borrowers", "update-requests"]),
+              specific_params=("obrp", ["overdue-letters", "update-borrowers", "update-requests",
+                                        "add-physical-copies-shelf-number-to-marc"]),
               task_submit_elaborate_specific_parameter_fnc=task_submit_elaborate_specific_parameter,
               version=__revision__,
               task_run_fnc = task_run_core