diff --git a/.circleci/config.yml b/.circleci/config.yml index ecf0df0..8b5f311 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -12,7 +12,7 @@ jobs: command: | sudo apt-get -qq update -y sudo apt-get -q install -y curl ca-certificates wget make - (cd ..; git clone https://github.com/antirez/redis.git; cd redis; git checkout 5.0.8; make -j$(nproc); sudo make install) + (cd ..; git clone https://github.com/antirez/redis.git; cd redis; git checkout 6.2.1; make -j$(nproc); sudo make install) SUDO=sudo make setup - run: name: Test diff --git a/Makefile b/Makefile index 85335b0..e11db65 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,17 @@ +define HELP +make start # Start containers +make stop # Stop containers +make clean # Remove demo containers and images +make test # Run test + VERBOSE=1 # Show more detailed information +make camera # Start camera process +make setup # Install prerequisites + +endef + +#---------------------------------------------------------------------------------------------- + start: @docker-compose up -d @@ -8,20 +21,26 @@ stop: build: @docker-compose build +clean: + @./tests/cats-n-dogs.sh clean + test: @./tests/cats-n-dogs.sh +show-logs: + @./tests/cats-n-dogs.sh logs + ifdef REDIS CAMERA_ARG=-u $(REDIS) endif camera: ifneq ($(VENV),0) - python2 -m virtualenv venv + python3 -m virtualenv venv . ./venv/bin/activate; pip install -r camera/requirements.txt . ./venv/bin/activate; python camera/read_camera.py $(CAMERA_ARG) else - python2 camera/read_camera.py $(CAMERA_ARG) + python3 camera/read_camera.py $(CAMERA_ARG) endif setup: @@ -31,4 +50,19 @@ setup: cd /tmp; tar xf git-lfs.tar.gz; $(SUDO) ./install.sh @git lfs pull -.PHONY: start stop build test camera setup +.PHONY: start stop build clean test camera setup help + +#---------------------------------------------------------------------------------------------- + +ifneq ($(HELP),) +ifneq ($(filter help,$(MAKECMDGOALS)),) +HELPFILE:=$(shell mktemp /tmp/make.help.XXXX) +endif +endif + +help: + $(file >$(HELPFILE),$(HELP)) + @echo + @cat $(HELPFILE) + @echo + @-rm -f $(HELPFILE) diff --git a/app/Dockerfile b/app/Dockerfile index fb31909..df1ab14 100644 --- a/app/Dockerfile +++ b/app/Dockerfile @@ -1,8 +1,8 @@ # OSNICK=stretch|bionic|buster -ARG OSNICK=buster +ARG OSNICK=bionic #---------------------------------------------------------------------------------------------- -FROM redisfab/redisedgevision-${OSNICK}:0.2.0 +FROM redisfab/redisedge:0.4.0-x64-${OSNICK} # This is due on the following error on ARMv8: # /usr/lib/aarch64-linux-gnu/libgomp.so.1: cannot allocate memory in static TLS block @@ -14,13 +14,12 @@ ENV DEBIAN_FRONTEND=noninteractive RUN apt-get -qq update RUN set -ex ;\ - apt-get install -y wget python3-distutils ;\ - wget -q https://bootstrap.pypa.io/get-pip.py -O /tmp/get-pip.py ;\ - python3 /tmp/get-pip.py + apt-get install -y python3 python3-pip ;\ + python3 -m pip install --upgrade pip WORKDIR /app ADD . /app -RUN pip3 install -r requirements.txt +RUN python3 -m pip install -r requirements.txt ENTRYPOINT [ "python3" ] diff --git a/app/gear.py b/app/gear.py index b602dfe..f515a09 100644 --- a/app/gear.py +++ b/app/gear.py @@ -27,6 +27,7 @@ def toOneList(l): return res def addToGraphRunner(x): + x = x['value'] try: xlog('addToGraphRunner:', 'count=', x['count']) @@ -54,7 +55,7 @@ def addToGraphRunner(x): animal = index[str(res.index(res1[0]) - 1)][1] xlog('addToGraphRunner:', 'animal=', animal) - return (animal, x['img']) + return animal, x['img'] except: xlog('addToGraphRunner: error:', sys.exc_info()[0]) @@ -62,21 +63,24 @@ def addToStream(x): # save animal name into a new stream try: redisgears.executeCommand('xadd', 'cats', 'MAXLEN', '~', str(MAX_IMAGES), '*', 'image', 'data:image/jpeg;base64,' + base64.b64encode(x[1]).decode('utf8')) + xlog('addToStream: ', x[0]) except: xlog('addToStream: error:', sys.exc_info()[0]) def shouldTakeFrame(x): + v = x['value'] try: global framesToDrop framesToDrop += 1 - xlog('shouldTakeFrame', x['count'], (framesToDrop % 10 == 0)) + xlog('shouldTakeFrame', v['count'], (framesToDrop % 10 == 0)) return framesToDrop % 10 == 0 except: xlog('shouldTakeFrame: error:', sys.exc_info()[0]) def passAll(x): + v = x['value'] try: - redisgears.executeCommand('xadd', 'all', 'MAXLEN', '~', str(MAX_IMAGES), '*', 'image', 'data:image/jpeg;base64,' + base64.b64encode(x['img']).decode('utf8')) + redisgears.executeCommand('xadd', 'all', 'MAXLEN', '~', str(MAX_IMAGES), '*', 'image', 'data:image/jpeg;base64,' + base64.b64encode(v['img']).decode('utf8')) except: xlog('passAll: error:', sys.exc_info()[0]) @@ -87,4 +91,4 @@ def passAll(x): map(addToGraphRunner).\ filter(lambda x: 'cat' in x[0]).\ foreach(addToStream).\ - register('camera:0') + register(prefix='camera:0') diff --git a/app/init.py b/app/init.py index 09f77bc..b8b3b0f 100644 --- a/app/init.py +++ b/app/init.py @@ -6,6 +6,8 @@ # Parse arguments parser = argparse.ArgumentParser() parser.add_argument('-u', '--url', help='Redis URL', type=str, default='redis://127.0.0.1:6379') + parser.add_argument('--with-requirements', action="store_true", help="Present requirements to Gears") + parser.add_argument('--no-requirements', action="store_true", help="Do not present requirements to Gears") args = parser.parse_args() # Set up some vars @@ -27,16 +29,31 @@ print('Loading model - ', end='') with open('models/mobilenet_v2_1.4_224_frozen.pb', 'rb') as f: model = f.read() - res = conn.execute_command('AI.MODELSET', 'mobilenet:model', 'TF', 'CPU', 'INPUTS', 'input', 'OUTPUTS', 'MobilenetV2/Predictions/Reshape_1', model) + res = conn.execute_command('AI.MODELSET', 'mobilenet:model', 'TF', 'CPU', 'INPUTS', 'input', 'OUTPUTS', 'MobilenetV2/Predictions/Reshape_1', 'BLOB', model) print(res) # Load the gear print('Loading gear - ', end='') with open('gear.py', 'rb') as f: gear = f.read() - res = conn.execute_command('RG.PYEXECUTE', gear) + if not args.no_requirements: + res = conn.execute_command('RG.PYEXECUTE', gear, 'REQUIREMENTS', 'imageio', 'numpy', 'opencv-python') + else: + res = conn.execute_command('RG.PYEXECUTE', gear) print(res) + while True: + res = conn.execute_command('RG.PYDUMPREQS') + finished = True + for i in range(len(res)): + dep = res[i] + downloaded_dep = dep[5] + finished = finished and (downloaded_dep == b'yes') + if finished: + print('gear loaded') + break + + # Lastly, set a key that indicates initialization has been performed print('Flag initialization as done - ', end='') print(conn.set(initialized_key, 'miauw')) diff --git a/camera/Dockerfile.debian b/camera/Dockerfile.debian index 9e3f210..9401bcb 100644 --- a/camera/Dockerfile.debian +++ b/camera/Dockerfile.debian @@ -1,25 +1,25 @@ # OSNICK=stretch|bionic|buster -ARG OSNICK=buster +ARG OSNICK=bionic #---------------------------------------------------------------------------------------------- -FROM redisfab/redisedgevision-${OSNICK}:0.2.0 +FROM redisfab/redisedge:0.4.0-x64-${OSNICK} # This is due on the following error on ARMv8: -# /usr/lib/aarch64-linux-gnu/libgomp.so.1: cannot allocate memory in static TLS block +# /usr/lib/aarch64-linux-gnu/libgomp.so.1: cannot allocate memory in static TLS block # Something is exausting TLS, causing libgomp to fail. Preloading it as a workaround helps. # ENV LD_PRELOAD /usr/lib/aarch64-linux-gnu/libgomp.so.1 ENV DEBIAN_FRONTEND=noninteractive RUN apt-get -qq update +RUN apt-get install -y wget python3 python3-pip ffmpeg libsm6 libxext6 -RUN set -x; \ - apt-get install -y wget python3-distutils ;\ - wget -q https://bootstrap.pypa.io/get-pip.py -O /tmp/get-pip.py ;\ - python3 /tmp/get-pip.py - -RUN pip install redis==3.2.1 +# wget -q https://bootstrap.pypa.io/get-pip.py -O /tmp/get-pip.py ;\ +# python3 /tmp/get-pip.py +ADD requirements.txt ./ +RUN python3 -m pip install --upgrade pip +RUN python3 -m pip install -r requirements.txt WORKDIR /usr/src/app ADD read_camera.py ./ diff --git a/camera/read_camera.py b/camera/read_camera.py index 2d9a2b6..5926042 100644 --- a/camera/read_camera.py +++ b/camera/read_camera.py @@ -62,7 +62,7 @@ def __len__(self): print('Connected to Redis') sys.stdout.flush() - if args.test is None: + if args.test is False: print('Operating in camera mode') sys.stdout.flush() if args.infile is None: diff --git a/camera/requirements.txt b/camera/requirements.txt index f209881..f9dbe38 100644 --- a/camera/requirements.txt +++ b/camera/requirements.txt @@ -1,3 +1,3 @@ -opencv-python>4.0,<4.1 +opencv-python redis==3.2.1 numpy>=1.17.1 diff --git a/redis/Dockerfile b/redis/Dockerfile index fa6b21d..11ceddf 100755 --- a/redis/Dockerfile +++ b/redis/Dockerfile @@ -1,9 +1,9 @@ # OSNICK=stretch|bionic|buster -ARG OSNICK=buster +ARG OSNICK=bionic #---------------------------------------------------------------------------------------------- -FROM redisfab/redisedgevision-${OSNICK}:0.2.0 +FROM redisfab/redisedge:0.4.0-x64-${OSNICK} # This is due on the following error on ARMv8: # /usr/lib/aarch64-linux-gnu/libgomp.so.1: cannot allocate memory in static TLS block @@ -17,9 +17,12 @@ WORKDIR /data EXPOSE 6379 ENTRYPOINT ["redis-server"] +RUN apt-get update +RUN apt-get install ffmpeg libsm6 libxext6 -y + CMD ["--loadmodule", "/usr/lib/redis/modules/redisai.so", \ - "--loadmodule", "/usr/lib/redis/modules/redisgears.so", \ - "PythonHomeDir", "/opt/redislabs/lib/modules/python3/"] + "--loadmodule", "/usr/lib/redis/modules/redisgears.so", \ + "PythonHomeDir", "/opt/redislabs/lib/modules/python3/"] # ADD redisedge.conf /etc diff --git a/tests/cats-n-dogs.sh b/tests/cats-n-dogs.sh index d1fa0fb..9efc5ed 100755 --- a/tests/cats-n-dogs.sh +++ b/tests/cats-n-dogs.sh @@ -32,13 +32,15 @@ start() { } stop() { + RMI_ARG="" + [[ $ALL == 1 ]] && RMI_ARG="--rmi all" if [[ $VERBOSE == 1 ]]; then $HERE/show-stream-stat.sh $HERE/show-log.sh - docker-compose $SPEC down -v --remove-orphans + docker-compose $SPEC down $RMI_ARG -v --remove-orphans else # --rmi local - docker-compose $SPEC down -v --remove-orphans >> $DOCKER_LOG 2>&1 + docker-compose $SPEC down $RMI_ARG -v --remove-orphans >> $DOCKER_LOG 2>&1 fi } @@ -54,6 +56,10 @@ build() { fi } +clean() { + ALL=1 stop +} + show_logs() { docker-compose $SPEC logs $* ./show-log.sh @@ -64,7 +70,7 @@ cats_demo() { echo "Testing ${ANIMAL}s ..." start for ((i = 0; i < 3; i++)); do - sleep 5 + sleep 10 [[ $VERBOSE == 1 ]] && ./show-stream-stat.sh num_cats=$(count) if [[ $num_cats > 0 ]]; then @@ -84,7 +90,7 @@ cats_demo() { } help() { - echo "[ANIMAL=cat|dog] [VERBOSE=0|1] [REBUILD=0|1] $0 [start|stop|build|count|logs|help]" + echo "[ANIMAL=cat|dog] [VERBOSE=0|1] [REBUILD=0|1] $0 [start|stop|build|clean|count|logs|help]" } cmd=$1 @@ -98,6 +104,8 @@ elif [[ $cmd == stop ]]; then stop elif [[ $cmd == build ]]; then build +elif [[ $cmd == clean ]]; then + clean elif [[ $cmd == count ]]; then echo $(count) exit 0 diff --git a/tests/cats-n-dogs.yaml b/tests/cats-n-dogs.yaml index ed17644..5657d1c 100644 --- a/tests/cats-n-dogs.yaml +++ b/tests/cats-n-dogs.yaml @@ -1,6 +1,7 @@ version: '3' services: redis: + container_name: redis.catndogs build: context: ../redis ports: @@ -8,17 +9,21 @@ services: environment: - MAX_IMAGES app: + container_name: app.catndogs build: ../app depends_on: - redis command: ['init.py', '--url', 'redis://redis:6379'] camera: + container_name: camera.catndogs build: context: ../camera dockerfile: Dockerfile.debian depends_on: - redis + - app environment: - ANIMAL - MAX_IMAGES command: ['python3', '/usr/src/app/read_camera.py', '-u', 'redis://redis:6379', '--test'] + \ No newline at end of file diff --git a/tests/fold.py b/tests/fold.py index 1667056..bb1e07e 100755 --- a/tests/fold.py +++ b/tests/fold.py @@ -15,14 +15,14 @@ def to_int(x): fields = sys.argv[1].split(':') else: fields = [''] -fields = map(lambda x: to_int(x), fields) +n = len(fields) +# fields = {x: to_int(x) for x in fields} row = "" i = 0 -n = len(fields) for line in sys.stdin: s = line.strip() m = len(s) - c = fields[i] + c = to_int(fields[i]) i = i + 1 if c != -1: if m > c: