process json correctly

Author: mjpost

detok.py

@@ -39,10 +39,20 @@ def debpe(s):
         if line.startswith('{'):
             try:
                 jobj = json.loads(line)
-                for score, translation in zip(jobj['scores'], jobj['translations']):
-                    print('{:.3f}'.format(score), fixTokenization(detokenizer(debpe(translation).split())), sep='\t', flush=True)
+                if 'translations' in jobj:
+                    scores = jobj['scores']
+                    translations = jobj['translations']
+                else:
+                    scores = [jobj['score']]
+                    translations = [jobj['translation']]
+
+                for score, translation in zip(scores, translations):
+                    # print('{:.3f}'.format(score), fixTokenization(detokenizer(debpe(translation).split())), sep='\t', flush=True)
+                    print(fixTokenization(detokenizer(debpe(translation).split())), flush=True)
+
                 continue
             except Exception:
+                print("Exception encountered!")
                 pass
 
         # Run if JSON parsing not relevant or fails

paraphrase.sh

@@ -9,13 +9,14 @@ python3 $MODEL_PATH/custom_constraints.py \
     --BPE-vocab $BPE_VOCAB \
     --compute-factor \
 | \
-python3 -m sockeye.translate \
-                -m $MODEL_PATH \
-                --json-input \
-                --beam-size 20 \
-                --beam-prune 20 \
-                --batch-size 10 \
-                --use-cpu \
-                --quiet \
-| \
-. $MODEL_PATH/post.sh
+    python3 -m sockeye.translate \
+      -m $MODEL_PATH \
+      --json-input \
+      --output-type json \
+      --beam-size 20 \
+      --beam-prune 20 \
+      --batch-size 10 \
+      --device-ids 0 \
+      --disable-device-locking \
+      "$@" \
+| ${MODEL_PATH}/detok.py