update demo

.circleci/config.yml

@@ -12,7 +12,7 @@ jobs:
           command: |
             sudo apt-get -qq update -y
             sudo apt-get -q install -y curl ca-certificates wget make
-            (cd ..; git clone https://github.com/antirez/redis.git; cd redis; git checkout 5.0.8; make -j$(nproc); sudo make install)
+            (cd ..; git clone https://github.com/antirez/redis.git; cd redis; git checkout 6.2.1; make -j$(nproc); sudo make install)
             SUDO=sudo make setup
       - run:
           name: Test

Makefile

@@ -1,4 +1,17 @@
 
+define HELP
+make start    # Start containers
+make stop     # Stop containers
+make clean    # Remove demo containers and images
+make test     # Run test
+  VERBOSE=1     # Show more detailed information
+make camera   # Start camera process
+make setup    # Install prerequisites
+
+endef
+
+#----------------------------------------------------------------------------------------------
+
 start:
 	@docker-compose up -d
 
@@ -8,20 +21,26 @@ stop:
 build:
 	@docker-compose build
 
+clean:
+	@./tests/cats-n-dogs.sh clean
+
 test:
 	@./tests/cats-n-dogs.sh
 
+show-logs:
+	@./tests/cats-n-dogs.sh logs
+
 ifdef REDIS
 CAMERA_ARG=-u $(REDIS)
 endif
 
 camera:
 ifneq ($(VENV),0)
-	python2 -m virtualenv venv
+	python3 -m virtualenv venv
 	. ./venv/bin/activate; pip install -r camera/requirements.txt
 	. ./venv/bin/activate; python camera/read_camera.py $(CAMERA_ARG)
 else
-	python2 camera/read_camera.py $(CAMERA_ARG)
+	python3 camera/read_camera.py $(CAMERA_ARG)
 endif
 
 setup:
@@ -31,4 +50,19 @@ setup:
     cd /tmp; tar xf git-lfs.tar.gz; $(SUDO) ./install.sh
 	@git lfs pull
 
-.PHONY: start stop build test camera setup
+.PHONY: start stop build clean test camera setup help
+
+#----------------------------------------------------------------------------------------------
+
+ifneq ($(HELP),) 
+ifneq ($(filter help,$(MAKECMDGOALS)),)
+HELPFILE:=$(shell mktemp /tmp/make.help.XXXX)
+endif
+endif
+
+help:
+	$(file >$(HELPFILE),$(HELP))
+	@echo
+	@cat $(HELPFILE)
+	@echo
+	@-rm -f $(HELPFILE)

app/Dockerfile

@@ -1,8 +1,8 @@
 # OSNICK=stretch|bionic|buster
-ARG OSNICK=buster
+ARG OSNICK=bionic
 
 #----------------------------------------------------------------------------------------------
-FROM redisfab/redisedgevision-${OSNICK}:0.2.0
+FROM redisfab/redisedge:0.4.0-x64-${OSNICK}
 
 # This is due on the following error on ARMv8:
 # /usr/lib/aarch64-linux-gnu/libgomp.so.1: cannot allocate memory in static TLS block 
@@ -14,13 +14,12 @@ ENV DEBIAN_FRONTEND=noninteractive
 RUN apt-get -qq update
 
 RUN set -ex ;\
-	apt-get install -y wget python3-distutils ;\
-	wget -q https://bootstrap.pypa.io/get-pip.py -O /tmp/get-pip.py ;\
-	python3 /tmp/get-pip.py
+	apt-get install -y python3 python3-pip ;\
+	python3 -m pip install --upgrade pip
 
 WORKDIR /app
 ADD . /app
 
-RUN pip3 install -r requirements.txt
+RUN python3 -m pip install -r requirements.txt
 
 ENTRYPOINT [ "python3" ]

app/gear.py

@@ -27,6 +27,7 @@ def toOneList(l):
     return res
 
 def addToGraphRunner(x):
+    x = x['value']
     try:
         xlog('addToGraphRunner:', 'count=', x['count'])
 
@@ -54,29 +55,32 @@ def addToGraphRunner(x):
         animal = index[str(res.index(res1[0]) - 1)][1]
         xlog('addToGraphRunner:', 'animal=', animal)
 
-        return (animal, x['img'])
+        return animal, x['img']
     except:
         xlog('addToGraphRunner: error:', sys.exc_info()[0])
 
 def addToStream(x):
     # save animal name into a new stream
     try:
         redisgears.executeCommand('xadd', 'cats', 'MAXLEN', '~', str(MAX_IMAGES), '*', 'image', 'data:image/jpeg;base64,' + base64.b64encode(x[1]).decode('utf8'))
+        xlog('addToStream: ', x[0])
     except:
         xlog('addToStream: error:', sys.exc_info()[0])
 
 def shouldTakeFrame(x):
+    v = x['value']
     try:
         global framesToDrop
         framesToDrop += 1
-        xlog('shouldTakeFrame', x['count'], (framesToDrop % 10 == 0))
+        xlog('shouldTakeFrame', v['count'], (framesToDrop % 10 == 0))
         return framesToDrop % 10 == 0
     except:
         xlog('shouldTakeFrame: error:', sys.exc_info()[0])
 
 def passAll(x):
+    v = x['value']
     try:
-        redisgears.executeCommand('xadd', 'all', 'MAXLEN', '~', str(MAX_IMAGES), '*', 'image', 'data:image/jpeg;base64,' + base64.b64encode(x['img']).decode('utf8'))
+        redisgears.executeCommand('xadd', 'all', 'MAXLEN', '~', str(MAX_IMAGES), '*', 'image', 'data:image/jpeg;base64,' + base64.b64encode(v['img']).decode('utf8'))
     except:
         xlog('passAll: error:', sys.exc_info()[0])
 
@@ -87,4 +91,4 @@ def passAll(x):
     map(addToGraphRunner).\
     filter(lambda x: 'cat' in x[0]).\
     foreach(addToStream).\
-    register('camera:0')
+    register(prefix='camera:0')

app/init.py

@@ -6,6 +6,8 @@
     # Parse arguments
     parser = argparse.ArgumentParser()
     parser.add_argument('-u', '--url', help='Redis URL', type=str, default='redis://127.0.0.1:6379')
+    parser.add_argument('--with-requirements', action="store_true", help="Present requirements to Gears")
+    parser.add_argument('--no-requirements', action="store_true", help="Do not present requirements to Gears")
     args = parser.parse_args()
 
     # Set up some vars
@@ -27,16 +29,31 @@
     print('Loading model - ', end='')
     with open('models/mobilenet_v2_1.4_224_frozen.pb', 'rb') as f:
         model = f.read()
-        res = conn.execute_command('AI.MODELSET', 'mobilenet:model', 'TF', 'CPU', 'INPUTS', 'input', 'OUTPUTS', 'MobilenetV2/Predictions/Reshape_1', model)
+        res = conn.execute_command('AI.MODELSET', 'mobilenet:model', 'TF', 'CPU', 'INPUTS', 'input', 'OUTPUTS', 'MobilenetV2/Predictions/Reshape_1', 'BLOB', model)
         print(res)
 
     # Load the gear
     print('Loading gear - ', end='')
     with open('gear.py', 'rb') as f:
         gear = f.read()
-        res = conn.execute_command('RG.PYEXECUTE', gear)
+        if not args.no_requirements:
+            res = conn.execute_command('RG.PYEXECUTE', gear, 'REQUIREMENTS', 'imageio', 'numpy', 'opencv-python')
+        else:
+            res = conn.execute_command('RG.PYEXECUTE', gear)
         print(res)
 
+    while True:
+        res = conn.execute_command('RG.PYDUMPREQS')
+        finished = True
+        for i in range(len(res)):
+            dep = res[i]
+            downloaded_dep = dep[5]
+            finished = finished and (downloaded_dep == b'yes')
+        if finished:
+            print('gear loaded')
+            break
+
+
     # Lastly, set a key that indicates initialization has been performed
     print('Flag initialization as done - ', end='')
     print(conn.set(initialized_key, 'miauw'))

camera/Dockerfile.debian

@@ -1,25 +1,25 @@
 # OSNICK=stretch|bionic|buster
-ARG OSNICK=buster
+ARG OSNICK=bionic
 
 #----------------------------------------------------------------------------------------------
-FROM redisfab/redisedgevision-${OSNICK}:0.2.0
+FROM redisfab/redisedge:0.4.0-x64-${OSNICK}
 
 # This is due on the following error on ARMv8:
-# /usr/lib/aarch64-linux-gnu/libgomp.so.1: cannot allocate memory in static TLS block 
+# /usr/lib/aarch64-linux-gnu/libgomp.so.1: cannot allocate memory in static TLS block
 # Something is exausting TLS, causing libgomp to fail. Preloading it as a workaround helps.
 # ENV LD_PRELOAD /usr/lib/aarch64-linux-gnu/libgomp.so.1
 
 ENV DEBIAN_FRONTEND=noninteractive
 
 RUN apt-get -qq update
+RUN apt-get install -y wget python3 python3-pip ffmpeg libsm6 libxext6
 
-RUN set -x; \
-	apt-get install -y wget python3-distutils ;\
-    wget -q https://bootstrap.pypa.io/get-pip.py -O /tmp/get-pip.py ;\
-	python3 /tmp/get-pip.py
-
-RUN pip install redis==3.2.1
+# wget -q https://bootstrap.pypa.io/get-pip.py -O /tmp/get-pip.py ;\
+# python3 /tmp/get-pip.py
 
+ADD requirements.txt ./
+RUN python3 -m pip install --upgrade pip
+RUN python3 -m pip install -r requirements.txt
 WORKDIR /usr/src/app
 
 ADD read_camera.py ./

camera/read_camera.py

@@ -62,7 +62,7 @@ def __len__(self):
     print('Connected to Redis')
     sys.stdout.flush()
 
-    if args.test is None:
+    if args.test is False:
         print('Operating in camera mode')
         sys.stdout.flush()
         if args.infile is None:

camera/requirements.txt

@@ -1,3 +1,3 @@
-opencv-python>4.0,<4.1
+opencv-python
 redis==3.2.1
 numpy>=1.17.1

redis/Dockerfile

@@ -1,9 +1,9 @@
 
 # OSNICK=stretch|bionic|buster
-ARG OSNICK=buster
+ARG OSNICK=bionic
 
 #----------------------------------------------------------------------------------------------
-FROM redisfab/redisedgevision-${OSNICK}:0.2.0
+FROM redisfab/redisedge:0.4.0-x64-${OSNICK}
 
 # This is due on the following error on ARMv8:
 # /usr/lib/aarch64-linux-gnu/libgomp.so.1: cannot allocate memory in static TLS block 
@@ -17,9 +17,12 @@ WORKDIR /data
 EXPOSE 6379
 ENTRYPOINT ["redis-server"]
 
+RUN apt-get update
+RUN apt-get install ffmpeg libsm6 libxext6  -y
+
 CMD ["--loadmodule", "/usr/lib/redis/modules/redisai.so", \
-	 "--loadmodule", "/usr/lib/redis/modules/redisgears.so", \
-	 "PythonHomeDir", "/opt/redislabs/lib/modules/python3/"]
+	"--loadmodule", "/usr/lib/redis/modules/redisgears.so", \
+	"PythonHomeDir", "/opt/redislabs/lib/modules/python3/"]
 
 # ADD redisedge.conf /etc
 

tests/cats-n-dogs.sh

@@ -32,13 +32,15 @@ start() {
 }
 
 stop() {
+	RMI_ARG=""
+	[[ $ALL == 1 ]] && RMI_ARG="--rmi all"
 	if [[ $VERBOSE == 1 ]]; then
 		$HERE/show-stream-stat.sh
 		$HERE/show-log.sh
-		docker-compose $SPEC down -v --remove-orphans
+		docker-compose $SPEC down $RMI_ARG -v --remove-orphans
 	else
 		#  --rmi local
-		docker-compose $SPEC down -v --remove-orphans >> $DOCKER_LOG 2>&1
+		docker-compose $SPEC down $RMI_ARG -v --remove-orphans >> $DOCKER_LOG 2>&1
 	fi
 }
 
@@ -54,6 +56,10 @@ build() {
 	fi
 }
 
+clean() {
+	ALL=1 stop
+}
+
 show_logs() {
 	docker-compose $SPEC logs $*
 	./show-log.sh
@@ -64,7 +70,7 @@ cats_demo() {
 	echo "Testing ${ANIMAL}s ..."
 	start
 	for ((i = 0; i < 3; i++)); do
-		sleep 5
+		sleep 10
 		[[ $VERBOSE == 1 ]] && ./show-stream-stat.sh
 		num_cats=$(count)
 		if [[ $num_cats > 0 ]]; then
@@ -84,7 +90,7 @@ cats_demo() {
 }
 
 help() {
-	echo "[ANIMAL=cat|dog] [VERBOSE=0|1] [REBUILD=0|1] $0 [start|stop|build|count|logs|help]"
+	echo "[ANIMAL=cat|dog] [VERBOSE=0|1] [REBUILD=0|1] $0 [start|stop|build|clean|count|logs|help]"
 }
 
 cmd=$1
@@ -98,6 +104,8 @@ elif [[ $cmd == stop ]]; then
 	stop
 elif [[ $cmd == build ]]; then
 	build
+elif [[ $cmd == clean ]]; then
+	clean
 elif [[ $cmd == count ]]; then
 	echo $(count)
 	exit 0

tests/cats-n-dogs.yaml

@@ -1,24 +1,29 @@
 version: '3'
 services:
   redis:
+    container_name: redis.catndogs
     build:
       context: ../redis
     ports:
       - "6379:6379"
     environment:
       - MAX_IMAGES
   app:
+    container_name: app.catndogs
     build: ../app
     depends_on:
       - redis
     command: ['init.py', '--url', 'redis://redis:6379']
   camera:
+    container_name: camera.catndogs
     build:
       context: ../camera
       dockerfile: Dockerfile.debian
     depends_on:
       - redis
+      - app
     environment:
       - ANIMAL
       - MAX_IMAGES
     command: ['python3', '/usr/src/app/read_camera.py', '-u', 'redis://redis:6379', '--test']
+

tests/fold.py

@@ -15,14 +15,14 @@ def to_int(x):
 	fields = sys.argv[1].split(':')
 else:
 	fields = ['']
-fields = map(lambda x: to_int(x), fields)
+n = len(fields)
+# fields = {x: to_int(x) for x in fields}
 row = ""
 i = 0
-n = len(fields)
 for line in sys.stdin:
 	s = line.strip()
 	m = len(s)
-	c = fields[i]
+	c = to_int(fields[i])
 	i = i + 1
 	if c != -1:
 		if m > c: