Merge pull request #282 from PaddlePaddle/master

merge master to v2.2.0 branch

Author: JingyuanZhang

e2e/server.ts

@@ -43,8 +43,8 @@ const ocr = new ConfigInfo('ocr', ocrWebpackConfig, true);
     if (instance.isModelSdk) {
         config.resolve = Object.assign({}, config.resolve, {
             alias: {
-                '@paddlejs/paddlejs-core': path.resolve(DIST_DIR, './core_bundle.js'),
-                '@paddlejs/paddlejs-backend-webgl': path.resolve(DIST_DIR, './webgl_bundle.js')
+                '@paddlejs/paddlejs-core': path.resolve(__dirname, '../packages/paddlejs-core/src/'),
+                '@paddlejs/paddlejs-backend-webgl': path.resolve(__dirname, '../packages/paddlejs-backend-webgl/src/')
             }
         });
     }

packages/paddlejs-core/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@paddlejs/paddlejs-core",
-  "version": "2.1.7",
+  "version": "2.1.9",
   "description": "",
   "main": "lib/index",
   "scripts": {

packages/paddlejs-core/src/commons/interface.ts

@@ -59,8 +59,14 @@ export interface FeedShape {
     fw: number;
     fh: number;
 };
+
+interface ModelObj {
+    model: Model;
+    params: Float32Array
+}
 export interface RunnerConfig {
-    modelPath: string;
+    modelPath?: string;
+    modelObj?: ModelObj;
     modelName?: string;
     feedShape?: FeedShape;
     fill?: string; // 缩放后用什么颜色填充不足方形部分

packages/paddlejs-core/src/loader.ts

@@ -70,7 +70,7 @@ export default class ModelLoader {
         if (this.separateChunk) {
             if (this.dataType === 'binary') {
                 await this.fetchChunks().then(allChunksData =>
-                    this.traverse(modelInfo.vars, allChunksData)
+                    ModelLoader.allocateParamsVar(modelInfo.vars, allChunksData)
                 );
             }
         }
@@ -105,9 +105,7 @@ export default class ModelLoader {
                 this.fetchOneChunk(this.urlConf.dir + this.getFileName(i))
             );
         }
-        // console.time('加载时间');
         return Promise.all(chunkArray).then(chunks => {
-            // console.timeEnd('加载时间');
             let chunksLength = 0;
             const f32Array: any[] = [];
             let float32Chunk;
@@ -128,7 +126,7 @@ export default class ModelLoader {
         });
     }
 
-    traverse(vars, allChunksData: Float32Array) {
+    static allocateParamsVar(vars, allChunksData: Float32Array) {
         let marker = 0; // 读到哪个位置了
         let len; // 当前op长度
         traverseVars(vars, item => {

packages/paddlejs-core/src/runner.ts

@@ -74,9 +74,19 @@ export default class Runner {
     }
 
     async load() {
-        const { modelPath } = this.runnerConfig;
-        const loader = new Loader(modelPath);
-        this.model = await loader.load();
+        const { modelPath, modelObj = null } = this.runnerConfig;
+        if (modelPath) {
+            const loader = new Loader(modelPath);
+            this.model = await loader.load();
+        }
+        else if (modelObj?.model && modelObj?.params) {
+            const {
+                model,
+                params
+            } = modelObj;
+            Loader.allocateParamsVar(model.vars, params);
+            this.model = model;
+        }
     }
 
     genGraph() {
@@ -373,8 +383,8 @@ export default class Runner {
 
         if (env.get('debug')
             && op.opData?.outputTensors
-            && op.opData.outputTensors[0]
-            && op.opData.outputTensors[0].tensorId === this.modelName + '_'
+            && op.opData.outputTensors[op.opData.outputTensors.length - 1]
+            && op.opData.outputTensors[op.opData.outputTensors.length - 1].tensorId === this.modelName + '_'
                 + (env.get('ns').layerName || env.get('layerName'))) {
             console.info(op.opData.name + '_' + op.opData.iLayer, 'runner op');
             return;

packages/paddlejs-examples/gesture/README.md

@@ -0,0 +1,33 @@
+[中文版](./README_cn.md)
+
+# gesture
+
+gesture is a gesture recognition module, including two models: gesture_detect and gesture_rec. gesture_detect model is used to identify the palm area of the person in the image. gesture_rec model is used to recognize human gesture. The interface provided by the module is simple, users only need to pass in gesture images to get the results.
+
+# Setup
+```
+npm install
+```
+
+### Compiles and hot-reloads for development
+```
+npm run dev
+```
+
+### Compiles and minifies for production
+```
+npm run build
+```
+
+# Usage
+
+```js
+import * as gesture from '@paddlejs-models/gesture';
+
+// Load gesture_detect model and gesture_rec model
+await gesture.load();
+
+// Get the image recognition results. The results include: palm frame coordinates and recognition results
+const res = await gesture.classify(img);
+
+```

packages/paddlejs-examples/gesture/README_cn.md

@@ -0,0 +1,33 @@
+[English](./README.md)
+
+# gesture
+
+gesture 为手势识别模块，包括两个模型：gesture_detect和gesture_rec。gesture_detect模型识别图片中人物手掌区域，gesture_rec模型识别人物手势。模块提供的接口简单，使用者只需传入手势图片即可获得结果。
+
+# 安装
+```
+npm install
+```
+
+### 编译
+```
+npm run dev
+```
+
+### 构建
+```
+npm run build
+```
+
+# 使用
+
+```js
+import * as gesture from '@paddlejs-models/gesture';
+
+// gesture_detect、gesture_rec模型加载
+await gesture.load();
+
+// 获取图片识别结果。结果包括：手掌框选坐标和识别结果
+const res = await gesture.classify(img);
+
+```

packages/paddlejs-examples/gesture/index.html

@@ -0,0 +1,102 @@
+<!DOCTYPE html>
+<html>
+<head>
+    <meta charset="utf-8">
+    <title>paddle ocr detection demo</title>
+    <meta name="viewport" content="width=device-width,minimum-scale=1.0,maximum-scale=1.0,user-scalable=no">
+    <style>
+        .wrapper {
+            position: relative;
+            display: flex;
+            text-align: left;
+            padding: 2%;
+            flex-wrap: wrap;
+            min-height: 500px;
+        }
+
+        .title {
+            font-weight: 600;
+            font-size: 20px;
+            margin-bottom: 10px;
+        }
+
+        .left {
+            display: flex;
+            flex-wrap: wrap;
+            width: min-content;
+            min-width: 55%;
+        }
+
+        .box {
+            min-width: 320px;
+            border: 1px solid #ddd;
+            border-radius: 5px;
+            margin-bottom: 100px;
+            padding: 10px;
+        }
+
+        .tool {
+            margin-right: 50px;
+            flex-shrink: 0;
+        }
+
+        .canvas {
+            width: max-content;
+            margin-right: 50px;
+        }
+
+        #image {
+            display: block;
+            margin-top: 10px;
+        }
+
+        #txt {
+            line-height: 2;
+            height: min-content;
+        }
+
+        #isLoading {
+            position: fixed;
+            top: 0;
+            left: 0;
+            right: 0;
+            bottom: 0;
+            width: 100vw;
+            height: 100vh;
+            background-color: rgba(0, 0, 0, .5);
+        }
+
+        #isLoading .loading-text {
+            color: #fff;
+            font-size: 24px;
+            text-align: center;
+            line-height: 100vh;
+        }
+
+    </style>
+</head>
+<body>
+<div class="wrapper">
+    <div class="left">
+        <div class="tool box">
+            <p class="title">上传手势图片：</p>
+            <input type="file" id="uploadImg" />
+            <img id="image" />
+        </div>
+        <div class="canvas box">
+            <p class="title">手掌位置框选：</p>
+            <canvas id="canvas"></canvas>
+        </div>
+    </div>
+    <div class="box">
+        <p class="title">手势识别结果：</p>
+        <div id="txt"></div>
+    </div>
+</div>
+
+<div id="isLoading">
+    <p class="loading-text center">loading中……</p>
+</div>
+
+</body>
+</html>

packages/paddlejs-examples/gesture/index.ts

@@ -0,0 +1,73 @@
+import * as gesture from '@paddlejs-models/gesture';
+
+const $uploadImg = document.getElementById('uploadImg') as HTMLInputElement;
+const $img = document.getElementById('image') as HTMLImageElement;
+const $txt = document.getElementById('txt');
+
+async function load() {
+    await gesture.load();
+    document.getElementById('isLoading').style.display = 'none';
+}
+
+async function run(input: HTMLImageElement) {
+    const res = await gesture.classify(input);
+    let text = '未识别到手';
+    if (res.box && res.box.length) {
+        text = res.type;
+        calculateBox(input, res.box);
+    }
+    $txt.innerText = text;
+}
+
+function calculateBox(img, box) {
+    let offsetX = 0;
+    let offsetY = 0;
+
+    if (img.width < img.height) {
+        offsetX = img.height - img.width;
+    }
+
+    if (img.width > img.height) {
+        offsetY = img.width - img.height;
+    }
+
+    const $canvas = document.getElementById('canvas') as HTMLCanvasElement;
+    const widthRatio = (img.width + offsetX) / 256;
+    const heightRatio = (img.height + offsetY) / 256;
+    const point = [];
+
+    box.forEach(item => {
+        const tmpPonit = [];
+        tmpPonit[0] = item[0] * widthRatio - offsetX / 2;
+        tmpPonit[1] = item[1] * heightRatio - offsetY / 2;
+        point.push(tmpPonit);
+    });
+    $canvas.width = img.width;
+    $canvas.height = img.height;
+    const ctx = $canvas.getContext('2d');
+    // 开始一个新的绘制路径
+    ctx.beginPath();
+    ctx.drawImage(img, 0, 0, $canvas.width, $canvas.height);
+    // 设置线条颜色为蓝色
+    ctx.strokeStyle = 'blue';
+    // 设置路径起点坐标
+    ctx.moveTo(point[0][0], point[0][1]);
+    ctx.lineTo(point[1][0], point[1][1]);
+    ctx.lineTo(point[2][0], point[2][1]);
+    ctx.lineTo(point[3][0], point[3][1]);
+    ctx.closePath();
+    ctx.stroke();
+}
+
+load();
+
+$uploadImg.onchange = (e: Event) => {
+    const reader = new FileReader();
+    reader.onload = () => {
+        $img.src = URL.createObjectURL((e.target as HTMLInputElement).files[0]);
+        $img.onload = () => {
+            run($img);
+        };
+    };
+    reader.readAsDataURL((e.target as HTMLInputElement).files[0]);
+};