Add the API to harvest each language resource

Author: lcahlander

src/main/js/frontend/package-lock.json

@@ -0,0 +1 @@
+<logs></logs>

src/main/xar-resources/data/log.xml

@@ -1,41 +1,44 @@
 xquery version "3.1";
 
-module namespace ll = "http://exist-db.org/xquery/stanford-nlp/load-language";
+import module namespace config = "http://exist-db.org/apps/stanford-nlp/config";
 
 import module namespace http = "http://expath.org/ns/http-client";
 import module namespace compression = "http://exist-db.org/xquery/compression";
-import module namespace console = "http://exist-db.org/xquery/console";
 import module namespace functx = "http://www.functx.com";
 import module namespace util = "http://exist-db.org/xquery/util";
 import module namespace map = "http://www.w3.org/2005/xpath-functions/map";
 import module namespace xmldb = "http://exist-db.org/xquery/xmldb";
 
-declare function ll:mkcol-recursive($collection, $components) {
+
+declare variable $local:language external;
+
+
+declare function local:mkcol-recursive($collection, $components) {
     if (exists($components)) then
         let $newColl := concat($collection, "/", $components[1])
         return (
-            if (xmldb:collection-available($newColl)) 
-            then () 
+            if (xmldb:collection-available($newColl))
+            then ()
             else xmldb:create-collection($collection, $components[1]),
-            ll:mkcol-recursive($newColl, subsequence($components, 2))
+            local:mkcol-recursive($newColl, subsequence($components, 2))
         )
     else
         ()
 };
 
-declare function ll:mkcol($collection, $path) {
-    ll:mkcol-recursive($collection, tokenize($path, "/"))
+declare function local:mkcol($collection, $path) {
+    local:mkcol-recursive($collection, tokenize($path, "/"))
 };
 
 
-declare function ll:entry-data($path as xs:anyURI, $type as xs:string, $data as item()?, $param as item()*) as item()?
+declare function local:entry-data($path as xs:anyURI, $type as xs:string, $data as item()?, $param as item()*) as item()?
 {
     let $path-before := functx:substring-before-last($path, "/")
     let $resource-name := functx:substring-after-last($path, "/")
-    let $coll := ll:mkcol("/db/apps/stanford-nlp/data", $path-before)
+    let $coll := local:mkcol("/db/apps/stanford-nlp/data", $path-before)
     let $decided :=
-            if (fn:ends-with($resource-name, ".properties")) 
-            then 
+            if (fn:ends-with($resource-name, ".properties"))
+            then
                 let $nl := "
"
                 let $lines := fn:tokenize(util:binary-to-string($data, "UTF-8"), $nl)
                 let $content := map:merge(
@@ -62,34 +65,51 @@ declare function ll:entry-data($path as xs:anyURI, $type as xs:string, $data as
                             else ()
                     )
                 return xmldb:store(
-                            "/db/apps/stanford-nlp/data/" || $path-before, 
-                            fn:replace($resource-name, ".properties", ".json"), 
+                            "/db/apps/stanford-nlp/data/" || $path-before,
+                            fn:replace($resource-name, ".properties", ".json"),
                             fn:serialize($content, map { "method": "json", "indent": true() })
-                        ) 
+                        )
             else ()
     let $stored := xmldb:store-as-binary("/db/apps/stanford-nlp/data/" || $path-before, $resource-name, $data)
-    let $log := console:log("item path=[" || $path || "] type=[" || $type || "]")
+    let $log := local:log("item path=[" || $path || "] type=[" || $type || "]")
     return ()
-}; 
+};
 
-declare function ll:entry-filter($path as xs:anyURI, $type as xs:string, $param as item()*) as xs:boolean
+declare function local:entry-filter($path as xs:anyURI, $type as xs:string, $param as item()*) as xs:boolean
 {
 	$type = "resource"
 };
 
-declare function ll:process($path as xs:string) {
-    let $log0 := console:log($path)
-    
+declare function local:log($message as xs:string)
+{
+    update insert
+        element { 'log' } {
+            attribute { 'timestamp' } { util:system-dateTime() },
+            attribute { 'language' } { $local:language },
+            $message
+        }
+    into
+        fn:doc("/db/apps/stanford-nlp/data/log.xml")//logs
+};
+
+declare function local:process($path as xs:string) {
+    let $log0 := local:log($path)
+
     let $req := <http:request href="{$path}" method="get"/>
-    
+
     let $zip := http:send-request($req)[2]
-    let $log := console:log(functx:atomic-type($zip))
-    
+    let $log := local:log(functx:atomic-type($zip))
+
     return compression:unzip(
-            $zip, 
-            util:function(xs:QName("ll:entry-filter"), 3),
-            (),  
-            util:function(xs:QName("ll:entry-data"), 4),
+            $zip,
+            util:function(xs:QName("local:entry-filter"), 3),
+            (),
+            util:function(xs:QName("local:entry-data"), 4),
             ()
         )
 };
+
+(local:log("start"),
+local:process($config:corenlp-model-url || $local:language || ".jar"),
+local:log("end")
+)

src/main/xar-resources/modules/load-languages.xq

@@ -0,0 +1,108 @@
+xquery version "3.1";
+
+module namespace api = "http://exist-db.org/xquery/stanford-nlp/api";
+import module namespace scheduler = "http://exist-db.org/xquery/scheduler";
+import module namespace map = "http://www.w3.org/2005/xpath-functions/map";
+
+declare namespace rest = "http://exquery.org/ns/restxq";
+declare namespace output = "http://www.w3.org/2010/xslt-xquery-serialization";
+
+
+declare function api:schedule-language($language as xs:string*) as map(*)
+{
+    let $a :=
+    scheduler:schedule-xquery-periodic-job(
+        "/db/apps/stanford-nlp/modules/load.xq",
+        500,
+        "nlp-load-" || $language || "-" || util:uuid(),
+        <parameters>
+            <param name="language" value="{$language}" />
+        </parameters>,
+        1000,
+        0
+    )
+    return
+        map {
+            "language": $language,
+            "status": $a
+        }
+};
+
+(:~
+  Start the loading of a language resource through a background process
+  @param $language The language to be loaded
+  @author Loren Cahlander
+  @version 1.0
+  @since 1.0
+  @custom:openapi-tag Natural Language Processing
+ :)
+declare
+%rest:GET
+%rest:path("/stanford/nlp/load/{$language}")
+%rest:produces("application/json")
+%output:media-type("application/json")
+%output:method("json")
+function api:load-language(
+$language as xs:string*
+) as map(*)
+{
+    switch ($language)
+    case "arabic"
+        return api:schedule-language($language)
+
+    case "chinese"
+        return api:schedule-language($language)
+
+    case "english"
+        return api:schedule-language($language)
+
+    case "english-kbp"
+        return api:schedule-language($language)
+
+    case "french"
+        return api:schedule-language($language)
+
+    case "german"
+        return api:schedule-language($language)
+
+    case "spanish"
+        return api:schedule-language($language)
+
+    default
+        return
+            map {
+                "status": fn:false(),
+                "languages": array {('arabic', 'chinese', 'english', 'english-kbp', 'french', 'german', 'spanish')}
+            }
+};
+
+(:~
+  Start the loading of a language resource through a background process
+  @author Loren Cahlander
+  @version 1.0
+  @since 1.0
+  @custom:openapi-tag Natural Language Processing
+ :)
+declare
+%rest:GET
+%rest:path("/stanford/nlp/logs")
+%rest:produces("application/json")
+%output:media-type("application/json")
+%output:method("json")
+function api:logs(
+) as array(*)
+{
+    array {
+        for $log in fn:doc("/db/apps/stanford-nlp/data/log.xml")//logs/log
+        let $timestamp := xs:string($log/@timestamp)
+        let $language := xs:string($log/@language)
+        let $text := $log/text()
+        order by $timestamp ascending
+        return
+            map {
+                'timestamp': $timestamp,
+                'language': $language,
+                'message': $text
+            }
+    }
+};

src/main/xar-resources/modules/load-language.xqm renamed to src/main/xar-resources/modules/load.xq

@@ -22,8 +22,27 @@ declare variable $dir external;
 (: the target collection into which the app is deployed :)
 declare variable $target external;
 
+let $modules :=    (
+"load.xq",
+()
+)
+
+let $a := (
+    for $module in $modules
+    return
+        (
+            sm:chown(xs:anyURI($target || "/modules/" || $module), "nlp:nlp"),
+            sm:chmod(xs:anyURI($target || "/modules/" || $module), "r-sr-xr-x"),
+            ()
+        ),
+    sm:chown(xs:anyURI($target || "/modules/rest-api.xqm"), "admin:dba"),
+    sm:chmod(xs:anyURI($target || "/modules/rest-api.xqm"), "r-sr-xr-x"),
+    ()
+)
+
 (:
   collection configuration was copied to the system config collection by pre-install.xq
   so we can now remove it from the app colllection
 :)
-xmldb:remove($target, "collection.xconf")
+return
+xmldb:remove($target, "collection.xconf")

src/main/xar-resources/modules/rest-api.xqm

@@ -24,6 +24,7 @@
 
     <prepare>pre-install.xq</prepare>
     <finish>post-install.xq</finish>
+    <permissions user="nlp" password="nlp" group="nlp" mode="rw-rw-r--"/>
 
     <!-- includes everything in src/main/xar-resources, README.md, and LICENSE -->
     <fileSets>