add spo dls to internal knowledge search

example-apps/internal-knowledge-search/app-ui/.env

@@ -1,3 +1,4 @@
-SEARCH_APP_NAME=some-search-application
-SEARCH_APP_API_KEY=xxxxxxxxxxxxxxxxxxx
-SEARCH_APP_ENDPOINT=https://some-search-end-point.co
+REACT_APP_SEARCH_APP_NAME=changeme
+REACT_APP_SEARCH_APP_ENDPOINT=http://localhost:9200
+REACT_APP_SEARCH_USER=elastic
+REACT_APP_SEARCH_PASSWORD=changeme

example-apps/internal-knowledge-search/app-ui/.gitignore

@@ -21,3 +21,6 @@
 npm-debug.log*
 yarn-debug.log*
 yarn-error.log*
+
+# secrets
+.env

example-apps/internal-knowledge-search/app-ui/README.md

@@ -106,5 +106,24 @@ To be able to use the index filtering and sorting in the UI you should update th
 
 ### Setting the environment variables
 
-You need to set `SEARCH_APP_NAME`, `SEARCH_APP_API_KEY` and `SEARCH_APP_ENDPOINT` inside [.env](.env) to the corresponding values, which you'll get when [creating a search application](https://www.elastic.co/guide/en/enterprise-search/current/search-applications.html).
+You need to set `REACT_APP_SEARCH_APP_NAME`, `REACT_APP_SEARCH_APP_USER`, `REACT_APP_SEARCH_APP_PASSWORD` and `REACT_APP_SEARCH_APP_ENDPOINT` inside [.env](.env) to the corresponding values, which you'll get when [creating a search application](https://www.elastic.co/guide/en/enterprise-search/current/search-applications.html).
 
+### Set up DLS with SPO
+1. create a connector in kibana named `search-sharepoint` 
+2. start connectors-python, if using connector clients
+3. enable DLS
+4. run an access control sync
+5. run a full sync
+6. define mappings, as above in this README
+7. create search application
+8. enable cors: https://www.elastic.co/guide/en/elasticsearch/reference/master/search-application-security.html#search-application-security-cors-elasticsearch
+
+
+### Make it go
+
+When it's time to actually start it up:
+
+```
+npm install
+npm run start
+```

example-apps/internal-knowledge-search/app-ui/package.json

@@ -3,7 +3,7 @@
   "version": "0.1.0",
   "private": true,
   "dependencies": {
-    "@elastic/search-application-client": "^8.9.0-1",
+    "@elastic/search-application-client": "file:elastic-search-application-client-8.9.0-2.tgz",
     "@fortawesome/react-fontawesome": "^0.2.0",
     "@fortawesome/free-solid-svg-icons": "^6.4.2",
     "@reduxjs/toolkit": "^1.9.7",

example-apps/internal-knowledge-search/app-ui/src/App.tsx

@@ -21,8 +21,8 @@ function App() {
                         <SidebarMenu/>
                         <main className="flex-1 p-6">
                             <Routes>
-                                <Route path="/" element={<SearchPage/>}/>
-                                <Route path="/settings" element={<SettingsPage/>}/>
+                                <Route path="/" element={<SettingsPage/>}/>
+                                <Route path="/search" element={<SearchPage/>}/>
                             </Routes>
                         </main>
                     </div>

example-apps/internal-knowledge-search/app-ui/src/components/SearchApplicationSettings.tsx

@@ -1,23 +1,222 @@
-import React from 'react';
+import React, {useEffect, useState} from 'react';
 import {useDispatch, useSelector} from "react-redux";
 import {updateSettings} from "../store/slices/searchApplicationSettingsSlice";
 import {useToast} from "../contexts/ToastContext";
 import {MessageType} from "./Toast";
 import {RootState} from "../store/store";
 
+
+
 const SearchApplicationSettings: React.FC = () => {
     const dispatch = useDispatch();
-    const {appName, apiKey, searchEndpoint} = useSelector((state: RootState) => state.searchApplicationSettings);
+    const {appName, appUser, appPassword, searchEndpoint, searchPersona, searchPersonaAPIKey} = useSelector((state: RootState) => state.searchApplicationSettings);
     const {showToast} = useToast();
 
+    const fetchPersonaOptions = async () => {
+        try {
+            const identitiesIndex = await fetchIdentitiesIndex(await fetchSearchApplicationIndices())
+            const identitiesPath = searchEndpoint + "/" + identitiesIndex + "/_search"
+            const response = await fetch(identitiesPath, {
+                method: "POST",
+                headers: {
+                    "Authorization": "Basic " + btoa(appUser + ":" + appPassword),
+                    "Content-Type": "application/json"
+                },
+                body: JSON.stringify({
+                    "size": 30 //TODO: fix magic number. This is just how many identities I have in my data
+                })
+            });
+            const jsonData = await response.json();
+            const ids = jsonData.hits.hits.map((hit) => hit._id)
+            return ids
+        } catch (e) {
+            console.log("Something went wrong tying to fetch ACL identities")
+            console.log(e)
+            return ["admin"]
+        }
+    }
+
+    const fetchSearchApplicationIndices = async () => {
+        try {
+            const searchApplicationPath = searchEndpoint + "/_application/search_application/" + appName
+            const response = await fetch(searchApplicationPath, {
+                headers: {
+                    "Authorization": "Basic " + btoa(appUser + ":" + appPassword)
+                }
+            })
+            const jsonData = await response.json();
+            const indices = jsonData.indices
+            return indices
+        } catch (e) {
+            console.log("Something went wrong trying to fetch the Search Application underlying indices")
+            console.log(e)
+            return []
+        }
+    }
+
+    const fetchIdentitiesIndex = async(applicationIndices) => {
+        try {
+            const identitiesIndexPath = searchEndpoint + "/.search-acl-filter*"
+            const response = await fetch(identitiesIndexPath, {
+                headers: {
+                    "Authorization": "Basic " + btoa(appUser + ":" + appPassword)
+                }
+            })
+            const jsonData = await response.json();
+            const identityIndices = Object.keys(jsonData)
+            const securedIndex = applicationIndices.find((applicationIndex) => identityIndices.includes(".search-acl-filter-"+applicationIndex))
+            return ".search-acl-filter-"+securedIndex
+        } catch (e) {
+            console.log("Something went wrong trying to fetch the Identities Index")
+            console.log(e)
+            return
+        }
+    }
+
+    const roleName = appName+"-key-role"
+    const defaultRoleDescriptor = {
+        [roleName]: {
+            "cluster": [],
+            "indices": [
+                {
+                    "names": [
+                        appName
+                    ],
+                    "privileges": [
+                        "read"
+                    ],
+                    "allow_restricted_indices": false
+                }
+            ],
+            "applications": [],
+            "run_as": [],
+            "metadata": {},
+            "transient_metadata": {
+                "enabled": true
+            },
+            "restriction": {
+                "workflows": [
+                    "search_application_query"
+                ]
+            }
+        }
+    }
+    const personaRoleDescriptor = async (persona) => {
+        const identitiesIndex = ".search-acl-filter-search-sharepoint" //TODO fix hardcoded
+        const identityPath = searchEndpoint + "/" + identitiesIndex + "/_doc/" + persona
+        const response = await fetch(identityPath, {headers: {"Authorization": "Basic " + btoa(appUser + ":" + appPassword)}});
+        const jsonData = await response.json();
+        const permissions = jsonData._source.query.template.params.access_control
+        return {
+            "dls-role": {
+                "cluster": ["all"],
+                "indices": [
+                    {
+                        "names": [appName],
+                        "privileges": ["read"],
+                        "query" : {
+                            "template": {
+                                "params": {
+                                    "access_control": permissions
+                                },
+                                "source" : `{
+                                    "bool": {
+                                        "filter": {
+                                            "bool": {
+                                                "should": [
+                                                    {
+                                                        "bool": {
+                                                            "must_not": {
+                                                                "exists": {
+                                                                    "field": "_allow_access_control"
+                                                                }
+                                                            }
+                                                        }
+                                                    },
+                                                    {
+                                                        "terms": {
+                                                            "_allow_access_control.enum": {{#toJson}}access_control{{/toJson}}
+                                                        }
+                                                    }
+                                                ]
+                                            }
+                                        }
+                                    }
+                                }`
+                            }
+                        }
+                    }
+                ],
+                "restriction": {
+                    "workflows": [
+                        "search_application_query"
+                    ]
+                }
+            }
+        }
+    }
+    const createPersonaAPIKey = async(persona) => {
+        const roleDescriptor = persona == "admin" ? defaultRoleDescriptor : await personaRoleDescriptor(persona)
+        const apiKeyPath = searchEndpoint + "/_security/api_key"
+        const response = await fetch(apiKeyPath, {
+            method: "POST",
+            headers: {
+                "Authorization": "Basic " + btoa(appUser + ":" + appPassword),
+                "Content-Type": "application/json",
+            },
+            body: JSON.stringify({
+                "name": appName+"-internal-knowledge-search-example-"+persona,
+                "expiration": "1h",
+                "role_descriptors": roleDescriptor,
+                "metadata": {
+                    "application": appName,
+                    "createdBy": appUser
+                }
+            })
+        })
+        const jsonData = await response.json()
+        return jsonData.encoded
+    }
+
+    const [searchPersonaOptions, setSearchPersonaOptions] = useState(["admin"]);
+
+    useEffect(()=>{
+        (async()=>{
+            const fetchedPersonas = await fetchPersonaOptions()
+            setSearchPersonaOptions(fetchedPersonas)
+            if (searchPersonaAPIKey == "missing") {
+                const createdAPIKey = await createPersonaAPIKey(searchPersona)
+                updateSearchPersonaAPIKey(createdAPIKey)
+            }
+        })()
+    },[])
+
+    const [isOpen, setIsOpen] = useState(false);
+
+    const toggleDropdown = () => {
+        setIsOpen(!isOpen);
+    };
+
+    const handlePersonaChange = async (value: string) => {
+        updateSearchPersona(value, await createPersonaAPIKey(value));
+    };
+
     const handleSave = () => {
-        dispatch(updateSettings({appName, apiKey, searchEndpoint}));
+        dispatch(updateSettings({appName, appUser, appPassword, searchEndpoint, searchPersona, searchPersonaAPIKey}));
         showToast("Settings saved!", MessageType.Info);
     };
 
-    const updateAppName = (e) => dispatch(updateSettings({appName: e.target.value, apiKey, searchEndpoint}))
-    const updateApiKey = (e) => dispatch(updateSettings({appName, apiKey: e.target.value, searchEndpoint}))
-    const updateSearchEndpoint = (e) => dispatch(updateSettings({appName, apiKey, searchEndpoint: e.target.value}))
+    const updateAppName = (e) => dispatch(updateSettings({appName: e.target.value, appUser, appPassword, searchEndpoint, searchPersona, searchPersonaAPIKey}))
+
+    const updateAppUser = (e) => dispatch(updateSettings({appName, appUser: e.target.value, appPassword, searchEndpoint, searchPersona, searchPersonaAPIKey}))
+
+    const updateAppPassword = (e) => dispatch(updateSettings({appName, appUser, appPassword: e.target.value, searchEndpoint, searchPersona, searchPersonaAPIKey}))
+
+    const updateSearchEndpoint = (e) => dispatch(updateSettings({appName, appUser, appPassword, searchEndpoint: e.target.value, searchPersona, searchPersonaAPIKey}))
+
+    const updateSearchPersona = (persona, apiKey) => dispatch(updateSettings({appName, appUser, appPassword, searchEndpoint, searchPersona: persona, searchPersonaAPIKey: apiKey}))
+
+    const updateSearchPersonaAPIKey = (apiKey) => dispatch(updateSettings({appName, appUser, appPassword, searchEndpoint, searchPersona, searchPersonaAPIKey: apiKey}))
 
     return (
         <div className="container mx-auto p-4 bg-white rounded shadow-md">
@@ -51,17 +250,34 @@ const SearchApplicationSettings: React.FC = () => {
 
             <div className="text-left mb-6 p-4 border rounded bg-gray-50">
                 <label
-                    htmlFor="apiKey"
+                    htmlFor="appUser"
                     className="block text-sm font-medium mb-1 text-gray-700"
                 >
-                    API Key:
+                    Application Elasticsearch Username:
                 </label>
-                <p className="text-xs mb-2 text-gray-500">Your unique API key used for authentication.</p>
+                <p className="text-xs mb-2 text-gray-500">The Elasticsearch username to use to establish a connection with.</p>
                 <input
-                    id="apiKey"
+                    id="appUser"
+                    type="text"
+                    value={appUser}
+                    onChange={updateAppUser}
+                    className="p-2 w-full border rounded focus:outline-none focus:shadow-outline"
+                />
+            </div>
+
+            <div className="text-left mb-6 p-4 border rounded bg-gray-50">
+                <label
+                    htmlFor="appPassword"
+                    className="block text-sm font-medium mb-1 text-gray-700"
+                >
+                    Application Elasticsearch Password:
+                </label>
+                <p className="text-xs mb-2 text-gray-500">The Elasticsearch password to use to establish a connection with.</p>
+                <input
+                    id="appPassword"
                     type="password"
-                    value={apiKey}
-                    onChange={updateApiKey}
+                    value={appPassword}
+                    onChange={updateAppPassword}
                     className="p-2 w-full border rounded focus:outline-none focus:shadow-outline"
                 />
             </div>
@@ -83,6 +299,33 @@ const SearchApplicationSettings: React.FC = () => {
                 />
             </div>
 
+            <div className="text-left mb-6 p-4 border rounded bg-gray-50">
+                <label
+                    htmlFor="searchPersona"
+                    className="block text-sm font-medium mb-1 text-gray-700"
+                >
+                    Search Persona:
+                </label>
+                <p className="text-xs mb-2 text-gray-500">The persona on whose behalf searches will be executed</p>
+                <div className="relative">
+                    <select
+                        onChange={ async (event) => await handlePersonaChange(event.target.value)}
+                        value={searchPersona}
+                        className="flex items-center space-x-2 p-2 bg-white rounded border border-gray-300 focus:outline-none focus:border-blue-500"
+                    >
+                        {searchPersonaOptions.includes(searchPersona) ? "" : <option value={searchPersona} key={searchPersona} className="block text-left p-2 hover:bg-gray-100 cursor-pointer">
+                            {searchPersona}
+                        </option>}
+                        {searchPersonaOptions.map((option, index) => (
+                            <option value={option} key={option} className="block text-left p-2 hover:bg-gray-100 cursor-pointer">
+                                {option}
+                            </option>
+                        ))}
+                    </select>
+                </div>
+            </div>
+
+
             <button
                 onClick={handleSave}
                 className="mt-4 bg-blue-500 hover:bg-blue-700 text-white font-bold py-2 px-4 rounded focus:outline-none focus:shadow-outline"